Removed hooks on DLL_PROCESS_DETACH

Fixes a crash when opening Blood 2's display settings from the launcher.

DDrawCompat/CompatGdi.cpp

@@ -196,6 +196,20 @@ namespace CompatGdi
 		return RealPrimarySurface::isFullScreen();
 	}
 
+	void unhookWndProc(LPCSTR className, WNDPROC oldWndProc)
+	{
+		HWND hwnd = CreateWindow(className, nullptr, 0, 0, 0, 0, 0, nullptr, nullptr, nullptr, 0);
+		SetClassLongPtr(hwnd, GCLP_WNDPROC, reinterpret_cast<LONG>(oldWndProc));
+		DestroyWindow(hwnd);
+	}
+
+	void uninstallHooks()
+	{
+		CompatGdiCaret::uninstallHooks();
+		CompatGdiWinProc::uninstallHooks();
+		CompatGdiPaintHandlers::uninstallHooks();
+	}
+
 	void updatePalette(DWORD startingEntry, DWORD count)
 	{
 		if (isEmulationEnabled() && CompatPrimarySurface::palette)

DDrawCompat/CompatGdi.h

@@ -13,6 +13,8 @@ namespace CompatGdi
 	void installHooks();
 	void invalidate(const RECT* rect);
 	bool isEmulationEnabled();
+	void unhookWndProc(LPCSTR className, WNDPROC oldWndProc);
+	void uninstallHooks();
 	void updatePalette(DWORD startingEntry, DWORD count);
 
 	extern CRITICAL_SECTION g_gdiCriticalSection;

DDrawCompat/CompatGdiCaret.cpp

@@ -10,6 +10,9 @@
 
 namespace
 {
+	HWINEVENTHOOK g_compatGdiCaretGeneralEventHook = nullptr;
+	HWINEVENTHOOK g_compatGdiCaretLocationChangeEventHook = nullptr;
+
 	template <typename Result, typename... Params>
 	using FuncPtr = Result(WINAPI *)(Params...);
 
@@ -121,18 +124,23 @@ namespace CompatGdiCaret
 	{
 		InitializeCriticalSection(&g_caretCriticalSection);
 
-		Compat::beginHookTransaction();
 		HOOK_GDI_CARET_FUNCTION(user32, CreateCaret);
 		HOOK_GDI_CARET_FUNCTION(user32, DestroyCaret);
 		HOOK_GDI_CARET_FUNCTION(user32, HideCaret);
 		HOOK_GDI_CARET_FUNCTION(user32, SetCaretPos);
 		HOOK_GDI_CARET_FUNCTION(user32, ShowCaret);
-		Compat::endHookTransaction();
 
 		const DWORD threadId = GetCurrentThreadId();
-		SetWinEventHook(EVENT_OBJECT_CREATE, EVENT_OBJECT_HIDE,
+		g_compatGdiCaretGeneralEventHook = SetWinEventHook(EVENT_OBJECT_CREATE, EVENT_OBJECT_HIDE,
-			nullptr, &compatGdiCaretEvent, 0, threadId, WINEVENT_OUTOFCONTEXT);
-		SetWinEventHook(EVENT_OBJECT_LOCATIONCHANGE, EVENT_OBJECT_LOCATIONCHANGE,
 			nullptr, &compatGdiCaretEvent, 0, threadId, WINEVENT_OUTOFCONTEXT);
+		g_compatGdiCaretLocationChangeEventHook = SetWinEventHook(
+			EVENT_OBJECT_LOCATIONCHANGE, EVENT_OBJECT_LOCATIONCHANGE, nullptr, &compatGdiCaretEvent,
+			0, threadId, WINEVENT_OUTOFCONTEXT);
+	}
+
+	void uninstallHooks()
+	{
+		UnhookWinEvent(g_compatGdiCaretLocationChangeEventHook);
+		UnhookWinEvent(g_compatGdiCaretGeneralEventHook);
 	}
 }

DDrawCompat/CompatGdiCaret.h

@@ -3,4 +3,5 @@
 namespace CompatGdiCaret
 {
 	void installHooks();
+	void uninstallHooks();
 }

DDrawCompat/CompatGdiDcFunctions.cpp

@@ -135,8 +135,6 @@ namespace CompatGdiDcFunctions
 {
 	void installHooks()
 	{
-		Compat::beginHookTransaction();
-
 		// Bitmap functions
 		HOOK_GDI_DC_FUNCTION(msimg32, AlphaBlend);
 		HOOK_GDI_DC_FUNCTION(gdi32, BitBlt);
@@ -226,7 +224,5 @@ namespace CompatGdiDcFunctions
 		// Undocumented functions
 		HOOK_GDI_DC_FUNCTION(gdi32, GdiDrawStream);
 		HOOK_GDI_DC_FUNCTION(gdi32, PolyPatBlt);
-
-		Compat::endHookTransaction();
 	}
 }

DDrawCompat/CompatGdiPaintHandlers.cpp

@@ -316,11 +316,18 @@ namespace CompatGdiPaintHandlers
 		CompatGdi::hookWndProc("#32768", g_origMenuWndProc, &menuWndProc);
 		CompatGdi::hookWndProc("ScrollBar", g_origScrollBarWndProc, &scrollBarWndProc);
 
-		Compat::beginHookTransaction();
 		HOOK_FUNCTION(user32, DefWindowProcA, defWindowProcA);
 		HOOK_FUNCTION(user32, DefWindowProcW, defWindowProcW);
 		HOOK_FUNCTION(user32, DefDlgProcA, defDlgProcA);
 		HOOK_FUNCTION(user32, DefDlgProcW, defDlgProcW);
-		Compat::endHookTransaction();
+	}
+
+	void uninstallHooks()
+	{
+		CompatGdi::unhookWndProc("ComboLBox", g_origComboListBoxWndProc);
+		CompatGdi::unhookWndProc("Edit", g_origEditWndProc);
+		CompatGdi::unhookWndProc("ListBox", g_origListBoxWndProc);
+		CompatGdi::unhookWndProc("#32768", g_origMenuWndProc);
+		CompatGdi::unhookWndProc("ScrollBar", g_origScrollBarWndProc);
 	}
 }

DDrawCompat/CompatGdiPaintHandlers.h

@@ -3,4 +3,5 @@
 namespace CompatGdiPaintHandlers
 {
 	void installHooks();
+	void uninstallHooks();
 }

DDrawCompat/CompatGdiScrollFunctions.cpp

@@ -47,10 +47,8 @@ namespace CompatGdiScrollFunctions
 {
 	void installHooks()
 	{
-		Compat::beginHookTransaction();
 		HOOK_FUNCTION(user32, ScrollWindow, scrollWindow);
 		HOOK_FUNCTION(user32, ScrollWindowEx, scrollWindowEx);
-		Compat::endHookTransaction();
 	}
 
 	void updateScrolledWindow(HWND hwnd)

DDrawCompat/CompatGdiWinProc.cpp

@@ -16,6 +16,8 @@
 
 namespace
 {
+	HHOOK g_callWndRetProcHook = nullptr;
+	HWINEVENTHOOK g_objectStateChangeEventHook = nullptr;
 	std::unordered_map<HWND, RECT> g_prevWindowRect;
 
 	void disableDwmAttributes(HWND hwnd);
@@ -196,8 +198,14 @@ namespace CompatGdiWinProc
 	void installHooks()
 	{
 		const DWORD threadId = GetCurrentThreadId();
-		SetWindowsHookEx(WH_CALLWNDPROCRET, callWndRetProc, nullptr, threadId);
+		g_callWndRetProcHook = SetWindowsHookEx(WH_CALLWNDPROCRET, callWndRetProc, nullptr, threadId);
-		SetWinEventHook(EVENT_OBJECT_STATECHANGE, EVENT_OBJECT_STATECHANGE,
+		g_objectStateChangeEventHook = SetWinEventHook(EVENT_OBJECT_STATECHANGE, EVENT_OBJECT_STATECHANGE,
 			nullptr, &objectStateChangeEvent, 0, threadId, WINEVENT_OUTOFCONTEXT);
 	}
+
+	void uninstallHooks()
+	{
+		UnhookWinEvent(g_objectStateChangeEventHook);
+		UnhookWindowsHookEx(g_callWndRetProcHook);
+	}
 }

DDrawCompat/CompatGdiWinProc.h

@@ -3,4 +3,5 @@
 namespace CompatGdiWinProc
 {
 	void installHooks();
+	void uninstallHooks();
 }

DDrawCompat/CompatRegistry.cpp

@@ -90,9 +90,7 @@ namespace CompatRegistry
 {
 	void installHooks()
 	{
-		Compat::beginHookTransaction();
 		HOOK_FUNCTION(KernelBase, RegGetValueW, regGetValueW);
-		Compat::endHookTransaction();
 	}
 
 	void setValue(HKEY key, const char* subKey, const char* valueName, DWORD value)

DDrawCompat/CompatVtable.h

@@ -114,9 +114,7 @@ private:
 				s_vtablePtrToCompatVtable[s_vtablePtr] = &s_compatVtable;
 				Compat::g_hookedMethods.emplace(origMethodPtr,
 					Compat::HookedMethodInfo(origMethodPtr, s_vtablePtrToCompatVtable));
-				Compat::beginHookTransaction();
 				Compat::hookFunction(origMethodPtr, newMethodPtr);
-				Compat::endHookTransaction();
 			}
 		}
 

DDrawCompat/DllMain.cpp

@@ -13,6 +13,7 @@
 #include "CompatVtable.h"
 #include "DDrawProcs.h"
 #include "DDrawRepository.h"
+#include "RealPrimarySurface.h"
 #include "Time.h"
 
 struct IDirectInput;
@@ -142,6 +143,10 @@ BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID /*lpvReserved*/)
 {
 	if (fdwReason == DLL_PROCESS_ATTACH)
 	{
+		char currentProcessPath[MAX_PATH] = {};
+		GetModuleFileName(nullptr, currentProcessPath, MAX_PATH);
+		Compat::Log() << "Process path: " << currentProcessPath;
+
 		char currentDllPath[MAX_PATH] = {};
 		GetModuleFileName(hinstDLL, currentDllPath, MAX_PATH);
 		Compat::Log() << "Loading DDrawCompat from " << currentDllPath;
@@ -181,8 +186,13 @@ BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID /*lpvReserved*/)
 	}
 	else if (fdwReason == DLL_PROCESS_DETACH)
 	{
+		Compat::Log() << "Detaching DDrawCompat";
+		RealPrimarySurface::removeUpdateThread();
+		CompatGdi::uninstallHooks();
+		Compat::unhookAllFunctions();
 		FreeLibrary(g_origDInputModule);
 		FreeLibrary(g_origDDrawModule);
+		Compat::Log() << "DDrawCompat detached successfully";
 	}
 
 	return TRUE;

DDrawCompat/Hook.cpp

@@ -1,5 +1,8 @@
 #define WIN32_LEAN_AND_MEAN
 
+#include <utility>
+#include <vector>
+
 #include <Windows.h>
 #include <detours.h>
 
@@ -8,6 +11,8 @@
 
 namespace
 {
+	std::vector<std::pair<void*, void*>> g_hookedFunctions;
+
 	FARPROC getProcAddress(HMODULE module, const char* procName)
 	{
 		if (!module || !procName)
@@ -45,25 +50,36 @@ namespace
 
 		return nullptr;
 	}
+
+	void hookFunction(const char* funcName, void*& origFuncPtr, void* newFuncPtr)
+	{
+		DetourTransactionBegin();
+		const bool attachSuccessful = NO_ERROR == DetourAttach(&origFuncPtr, newFuncPtr);
+		const bool commitSuccessful = NO_ERROR == DetourTransactionCommit();
+		if (!attachSuccessful || !commitSuccessful)
+		{
+			if (funcName)
+			{
+				Compat::Log() << "Failed to hook a function: " << funcName;
+			}
+			else
+			{
+				Compat::Log() << "Failed to hook a function: " << origFuncPtr;
+			}
+			return;
+		}
+
+		g_hookedFunctions.push_back(std::make_pair(origFuncPtr, newFuncPtr));
+	}
 }
 
 namespace Compat
 {
-	void beginHookTransaction()
-	{
-		DetourTransactionBegin();
-	}
-
-	void endHookTransaction()
-	{
-		DetourTransactionCommit();
-	}
-
 	void hookFunction(void*& origFuncPtr, void* newFuncPtr)
 	{
-		DetourAttach(&origFuncPtr, newFuncPtr);
+		::hookFunction(nullptr, origFuncPtr, newFuncPtr);
 	}
-
+	
 	void hookFunction(const char* moduleName, const char* funcName, void*& origFuncPtr, void* newFuncPtr)
 	{
 		FARPROC procAddr = getProcAddress(GetModuleHandle(moduleName), funcName);
@@ -74,10 +90,16 @@ namespace Compat
 		}
 
 		origFuncPtr = procAddr;
-		if (NO_ERROR != DetourAttach(&origFuncPtr, newFuncPtr))
+		::hookFunction(funcName, origFuncPtr, newFuncPtr);
+	}
+
+	void unhookAllFunctions()
+	{
+		for (auto& hookedFunc : g_hookedFunctions)
 		{
-			Compat::Log() << "Failed to hook a function: " << funcName;
+			DetourTransactionBegin();
-			return;
+			DetourDetach(&hookedFunc.first, hookedFunc.second);
+			DetourTransactionCommit();
 		}
 	}
 }

DDrawCompat/Hook.h

@@ -7,9 +7,6 @@
 
 namespace Compat
 {
-	void beginHookTransaction();
-	void endHookTransaction();
-
 	template <typename OrigFuncPtr, OrigFuncPtr origFunc>
 	OrigFuncPtr& getOrigFuncPtr()
 	{
@@ -26,4 +23,6 @@ namespace Compat
 		hookFunction(moduleName, funcName,
 			reinterpret_cast<void*&>(getOrigFuncPtr<OrigFuncPtr, origFunc>()), newFuncPtr);
 	}
+
+	void unhookAllFunctions();
 }

DDrawCompat/RealPrimarySurface.cpp

@@ -24,6 +24,7 @@ namespace
 	IDirectDrawSurface7* g_backBuffer = nullptr;
 	IReleaseNotifier g_releaseNotifier(onRelease);
 
+	bool g_stopUpdateThread = false;
 	HANDLE g_updateThread = nullptr;
 	HANDLE g_updateEvent = nullptr;
 	RECT g_updateRect = {};
@@ -140,6 +141,11 @@ namespace
 		{
 			WaitForSingleObject(g_updateEvent, INFINITE);
 
+			if (g_stopUpdateThread)
+			{
+				return 0;
+			}
+
 			const long long qpcTargetNextUpdate = g_qpcNextUpdate;
 			const int msUntilNextUpdate =
 				Time::qpcToMs(qpcTargetNextUpdate - Time::queryPerformanceCounter());
@@ -317,6 +323,25 @@ void RealPrimarySurface::release()
 	}
 }
 
+void RealPrimarySurface::removeUpdateThread()
+{
+	if (!g_updateThread)
+	{
+		return;
+	}
+
+	g_stopUpdateThread = true;
+	SetEvent(g_updateEvent);
+	if (WAIT_OBJECT_0 != WaitForSingleObject(g_updateThread, 1000))
+	{
+		TerminateThread(g_updateThread, 0);
+		Compat::Log() << "The update thread was terminated forcefully";
+	}
+	ResetEvent(g_updateEvent);
+	g_stopUpdateThread = false;
+	g_updateThread = nullptr;
+}
+
 HRESULT RealPrimarySurface::restore()
 {
 	return g_frontBuffer->lpVtbl->Restore(g_frontBuffer);

DDrawCompat/RealPrimarySurface.h

@@ -18,6 +18,7 @@ public:
 	static bool isFullScreen();
 	static bool isLost();
 	static void release();
+	static void removeUpdateThread();
 	static HRESULT restore();
 	static void setClipper(LPDIRECTDRAWCLIPPER clipper);
 	static void setPalette();