backup/DDrawCompat
1
0
Fork 0
mirror of https://github.com/narzoul/DDrawCompat synced 2024-12-30 08:55:36 +01:00
Code Issues Projects Releases Activity

Fixed crash when rehooking a user-mode display driver

Browse Source
This commit is contained in:
narzoul 2018-12-30 12:41:23 +01:00
parent c1b35e0434
commit d655947f18
9 changed files with 52 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
DDrawCompat/Common/CompatVtable.h
View File

@ -41,17 +41,25 @@ public:
} }
} }
static void hookDriverVtable(HANDLE context, const Vtable* vtable) static void hookDriverVtable(HMODULE module, HANDLE context, const Vtable* vtable)
{ {
if (vtable && s_origVtables.find(context) == s_origVtables.end()) if (!vtable)
{ {
HookVisitor<DriverHook> visitor(*vtable, s_origVtables[context]); return;
}
if (s_origModuleVtables.find(module) == s_origModuleVtables.end())
{
HookVisitor<DriverHook> visitor(*vtable, s_origModuleVtables[module]);
forEach<Vtable>(visitor); forEach<Vtable>(visitor);
} }
s_origVtables[context] = s_origModuleVtables[module];
} }
static Vtable s_origVtable; static Vtable s_origVtable;
static std::map<HANDLE, Vtable> s_origVtables; static std::map<HANDLE, Vtable> s_origVtables;
static std::map<HMODULE, Vtable> s_origModuleVtables;
static const Vtable* s_origVtablePtr; static const Vtable* s_origVtablePtr;
private: private:
@ -185,6 +193,9 @@ Vtable CompatVtable<Vtable>::s_origVtable = {};
template <typename Vtable> template <typename Vtable>
std::map<HANDLE, Vtable> CompatVtable<Vtable>::s_origVtables; std::map<HANDLE, Vtable> CompatVtable<Vtable>::s_origVtables;
template <typename Vtable>
std::map<HMODULE, Vtable> CompatVtable<Vtable>::s_origModuleVtables;
template <typename Vtable> template <typename Vtable>
const Vtable* CompatVtable<Vtable>::s_origVtablePtr = nullptr; const Vtable* CompatVtable<Vtable>::s_origVtablePtr = nullptr;

8
DDrawCompat/D3dDdi/AdapterFuncs.cpp
View File

@ -7,6 +7,7 @@
namespace namespace
{ {
std::map<HANDLE, D3DNTHAL_D3DEXTENDEDCAPS> g_d3dExtendedCaps; std::map<HANDLE, D3DNTHAL_D3DEXTENDEDCAPS> g_d3dExtendedCaps;
std::map<HANDLE, HMODULE> g_adapterModule;
HRESULT APIENTRY closeAdapter(HANDLE hAdapter) HRESULT APIENTRY closeAdapter(HANDLE hAdapter)
{ {
@ -15,6 +16,7 @@ namespace
{ {
D3dDdi::AdapterFuncs::s_origVtables.erase(hAdapter); D3dDdi::AdapterFuncs::s_origVtables.erase(hAdapter);
g_d3dExtendedCaps.erase(hAdapter); g_d3dExtendedCaps.erase(hAdapter);
g_adapterModule.erase(hAdapter);
} }
return result; return result;
} }
@ -26,7 +28,8 @@ namespace
hAdapter, pCreateData); hAdapter, pCreateData);
if (SUCCEEDED(result)) if (SUCCEEDED(result))
{ {
D3dDdi::DeviceFuncs::hookDriverVtable(pCreateData->hDevice, pCreateData->pDeviceFuncs); D3dDdi::DeviceFuncs::hookDriverVtable(
g_adapterModule[hAdapter], pCreateData->hDevice, pCreateData->pDeviceFuncs);
D3dDdi::DeviceFuncs::onCreateDevice(hAdapter, pCreateData->hDevice); D3dDdi::DeviceFuncs::onCreateDevice(hAdapter, pCreateData->hDevice);
} }
return result; return result;
@ -52,7 +55,7 @@ namespace D3dDdi
return it != g_d3dExtendedCaps.end() ? it->second : emptyCaps; return it != g_d3dExtendedCaps.end() ? it->second : emptyCaps;
} }
void AdapterFuncs::onOpenAdapter(HANDLE adapter) void AdapterFuncs::onOpenAdapter(HMODULE module, HANDLE adapter)
{ {
D3DNTHAL_D3DEXTENDEDCAPS d3dExtendedCaps = {}; D3DNTHAL_D3DEXTENDEDCAPS d3dExtendedCaps = {};
D3DDDIARG_GETCAPS getCaps = {}; D3DDDIARG_GETCAPS getCaps = {};
@ -62,6 +65,7 @@ namespace D3dDdi
D3dDdi::AdapterFuncs::s_origVtables.at(adapter).pfnGetCaps(adapter, &getCaps); D3dDdi::AdapterFuncs::s_origVtables.at(adapter).pfnGetCaps(adapter, &getCaps);
g_d3dExtendedCaps[adapter] = d3dExtendedCaps; g_d3dExtendedCaps[adapter] = d3dExtendedCaps;
g_adapterModule[adapter] = module;
} }
void AdapterFuncs::setCompatVtable(D3DDDI_ADAPTERFUNCS& vtable) void AdapterFuncs::setCompatVtable(D3DDDI_ADAPTERFUNCS& vtable)

2
DDrawCompat/D3dDdi/AdapterFuncs.h
View File

@ -14,7 +14,7 @@ namespace D3dDdi
{ {
public: public:
static const D3DNTHAL_D3DEXTENDEDCAPS& getD3dExtendedCaps(HANDLE adapter); static const D3DNTHAL_D3DEXTENDEDCAPS& getD3dExtendedCaps(HANDLE adapter);
static void onOpenAdapter(HANDLE adapter); static void onOpenAdapter(HMODULE module, HANDLE adapter);
static void setCompatVtable(D3DDDI_ADAPTERFUNCS& vtable); static void setCompatVtable(D3DDDI_ADAPTERFUNCS& vtable);
}; };
} }

4
DDrawCompat/D3dDdi/Device.cpp
View File

@ -162,7 +162,7 @@ namespace D3dDdi
if (SUCCEEDED(result)) if (SUCCEEDED(result))
{ {
m_oversizedResources.emplace(data.hResource, m_oversizedResources.emplace(data.hResource,
OversizedResource(m_adapter, m_device, data.Format, origSurfList[0])); OversizedResource(*m_origVtable, m_adapter, m_device, data.Format, origSurfList[0]));
} }
return result; return result;
@ -191,7 +191,7 @@ namespace D3dDdi
if (SUCCEEDED(result) && data.Flags.RenderTarget && !data.Flags.Primary && isVidMemPool(data.Pool)) if (SUCCEEDED(result) && data.Flags.RenderTarget && !data.Flags.Primary && isVidMemPool(data.Pool))
{ {
m_renderTargetResources.emplace(data.hResource, m_renderTargetResources.emplace(data.hResource,
RenderTargetResource(m_device, data.hResource, data.Format, data.SurfCount)); RenderTargetResource(*m_origVtable, m_device, data.hResource, data.Format, data.SurfCount));
} }
return result; return result;

13
DDrawCompat/D3dDdi/Hooks.cpp
View File

@ -26,6 +26,7 @@ namespace
{ {
UINT g_ddiVersion = 0; UINT g_ddiVersion = 0;
std::wstring g_hookedUmdFileName; std::wstring g_hookedUmdFileName;
HMODULE g_hookedUmdModule = nullptr;
PFND3DDDI_OPENADAPTER g_origOpenAdapter = nullptr; PFND3DDDI_OPENADAPTER g_origOpenAdapter = nullptr;
void hookOpenAdapter(const std::wstring& umdFileName); void hookOpenAdapter(const std::wstring& umdFileName);
@ -35,12 +36,12 @@ namespace
void hookOpenAdapter(const std::wstring& umdFileName) void hookOpenAdapter(const std::wstring& umdFileName)
{ {
g_hookedUmdFileName = umdFileName; g_hookedUmdFileName = umdFileName;
HMODULE module = LoadLibraryW(umdFileName.c_str()); g_hookedUmdModule = LoadLibraryW(umdFileName.c_str());
if (module) if (g_hookedUmdModule)
{ {
Compat::hookFunction(module, "OpenAdapter", Compat::hookFunction(g_hookedUmdModule, "OpenAdapter",
reinterpret_cast<void*&>(g_origOpenAdapter), &openAdapter); reinterpret_cast<void*&>(g_origOpenAdapter), &openAdapter);
FreeLibrary(module); FreeLibrary(g_hookedUmdModule);
} }
} }
@ -58,8 +59,8 @@ namespace
hookedUmdFileNames.insert(g_hookedUmdFileName); hookedUmdFileNames.insert(g_hookedUmdFileName);
} }
g_ddiVersion = min(pOpenData->Version, pOpenData->DriverVersion); g_ddiVersion = min(pOpenData->Version, pOpenData->DriverVersion);
D3dDdi::AdapterFuncs::hookDriverVtable(pOpenData->hAdapter, pOpenData->pAdapterFuncs); D3dDdi::AdapterFuncs::hookDriverVtable(g_hookedUmdModule, pOpenData->hAdapter, pOpenData->pAdapterFuncs);
D3dDdi::AdapterFuncs::onOpenAdapter(pOpenData->hAdapter); D3dDdi::AdapterFuncs::onOpenAdapter(g_hookedUmdModule, pOpenData->hAdapter);
} }
return LOG_RESULT(result); return LOG_RESULT(result);
} }

20
DDrawCompat/D3dDdi/OversizedResource.cpp
View File

@ -6,8 +6,10 @@
namespace D3dDdi namespace D3dDdi
{ {
OversizedResource::OversizedResource( OversizedResource::OversizedResource(
HANDLE adapter, HANDLE device, D3DDDIFORMAT format, const D3DDDI_SURFACEINFO& surfaceInfo) const D3DDDI_DEVICEFUNCS& deviceFuncs, HANDLE adapter, HANDLE device,
: m_adapter(adapter) D3DDDIFORMAT format, const D3DDDI_SURFACEINFO& surfaceInfo)
: m_deviceFuncs(deviceFuncs)
, m_adapter(adapter)
, m_device(device) , m_device(device)
, m_format(format) , m_format(format)
, m_surfaceInfo(surfaceInfo) , m_surfaceInfo(surfaceInfo)
@ -20,7 +22,7 @@ namespace D3dDdi
if (rect.right <= static_cast<LONG>(caps.dwMaxTextureWidth) && if (rect.right <= static_cast<LONG>(caps.dwMaxTextureWidth) &&
rect.bottom <= static_cast<LONG>(caps.dwMaxTextureHeight)) rect.bottom <= static_cast<LONG>(caps.dwMaxTextureHeight))
{ {
return D3dDdi::DeviceFuncs::s_origVtables.at(m_device).pfnBlt(m_device, &data); return m_deviceFuncs.pfnBlt(m_device, &data);
} }
HANDLE origResource = resource; HANDLE origResource = resource;
@ -33,14 +35,13 @@ namespace D3dDdi
rect = RECT{ 0, 0, rect.right - rect.left, rect.bottom - rect.top }; rect = RECT{ 0, 0, rect.right - rect.left, rect.bottom - rect.top };
} }
const auto& deviceFuncs = D3dDdi::DeviceFuncs::s_origVtables.at(m_device); HRESULT result = m_deviceFuncs.pfnBlt(m_device, &data);
HRESULT result = deviceFuncs.pfnBlt(m_device, &data);
if (bltResource) if (bltResource)
{ {
resource = origResource; resource = origResource;
rect = origRect; rect = origRect;
deviceFuncs.pfnDestroyResource(m_device, bltResource); m_deviceFuncs.pfnDestroyResource(m_device, bltResource);
} }
return result; return result;
@ -76,14 +77,13 @@ namespace D3dDdi
bltResourceData.pSurfList = &bltSurfaceInfo; bltResourceData.pSurfList = &bltSurfaceInfo;
bltResourceData.SurfCount = 1; bltResourceData.SurfCount = 1;
const auto& deviceFuncs = D3dDdi::DeviceFuncs::s_origVtables.at(m_device); if (m_deviceFuncs.pfnCreateResource2)
if (deviceFuncs.pfnCreateResource2)
{ {
deviceFuncs.pfnCreateResource2(m_device, &bltResourceData); m_deviceFuncs.pfnCreateResource2(m_device, &bltResourceData);
} }
else else
{ {
deviceFuncs.pfnCreateResource(m_device, m_deviceFuncs.pfnCreateResource(m_device,
reinterpret_cast<D3DDDIARG_CREATERESOURCE*>(&bltResourceData)); reinterpret_cast<D3DDDIARG_CREATERESOURCE*>(&bltResourceData));
} }
return bltResourceData.hResource; return bltResourceData.hResource;

3
DDrawCompat/D3dDdi/OversizedResource.h
View File

@ -8,7 +8,7 @@ namespace D3dDdi
class OversizedResource class OversizedResource
{ {
public: public:
OversizedResource(HANDLE adapter, HANDLE device, OversizedResource(const D3DDDI_DEVICEFUNCS& deviceFuncs, HANDLE adapter, HANDLE device,
D3DDDIFORMAT format, const D3DDDI_SURFACEINFO& surfaceInfo); D3DDDIFORMAT format, const D3DDDI_SURFACEINFO& surfaceInfo);
HRESULT bltFrom(D3DDDIARG_BLT data); HRESULT bltFrom(D3DDDIARG_BLT data);
@ -20,6 +20,7 @@ namespace D3dDdi
HRESULT blt(D3DDDIARG_BLT& data, HANDLE& resource, RECT& rect); HRESULT blt(D3DDDIARG_BLT& data, HANDLE& resource, RECT& rect);
HANDLE createBltResource(RECT bltRect); HANDLE createBltResource(RECT bltRect);
const D3DDDI_DEVICEFUNCS& m_deviceFuncs;
HANDLE m_adapter; HANDLE m_adapter;
HANDLE m_device; HANDLE m_device;
D3DDDIFORMAT m_format; D3DDDIFORMAT m_format;

13
DDrawCompat/D3dDdi/RenderTargetResource.cpp
View File

@ -4,9 +4,10 @@
namespace D3dDdi namespace D3dDdi
{ {
RenderTargetResource::RenderTargetResource( RenderTargetResource::RenderTargetResource(const D3DDDI_DEVICEFUNCS& deviceFuncs,
HANDLE device, HANDLE resource, D3DDDIFORMAT format, UINT surfaceCount) HANDLE device, HANDLE resource, D3DDDIFORMAT format, UINT surfaceCount)
: m_device(device) : m_deviceFuncs(deviceFuncs)
, m_device(device)
, m_resource(resource) , m_resource(resource)
, m_bytesPerPixel(getBytesPerPixel(format)) , m_bytesPerPixel(getBytesPerPixel(format))
, m_subResources(surfaceCount, SubResource(*this)) , m_subResources(surfaceCount, SubResource(*this))
@ -17,7 +18,7 @@ namespace D3dDdi
{ {
if (data.SubResourceIndex >= m_subResources.size()) if (data.SubResourceIndex >= m_subResources.size())
{ {
return D3dDdi::DeviceFuncs::s_origVtables.at(m_device).pfnLock(m_device, &data); return m_deviceFuncs.pfnLock(m_device, &data);
} }
auto& subResource = m_subResources[data.SubResourceIndex]; auto& subResource = m_subResources[data.SubResourceIndex];
@ -36,7 +37,7 @@ namespace D3dDdi
const UINT origFlags = data.Flags.Value; const UINT origFlags = data.Flags.Value;
data.Flags.Value = 0; data.Flags.Value = 0;
const HRESULT result = D3dDdi::DeviceFuncs::s_origVtables.at(m_device).pfnLock(m_device, &data); const HRESULT result = m_deviceFuncs.pfnLock(m_device, &data);
data.Flags.Value = origFlags; data.Flags.Value = origFlags;
if (SUCCEEDED(result)) if (SUCCEEDED(result))
@ -54,7 +55,7 @@ namespace D3dDdi
{ {
if (data.SubResourceIndex >= m_subResources.size()) if (data.SubResourceIndex >= m_subResources.size())
{ {
return D3dDdi::DeviceFuncs::s_origVtables.at(m_device).pfnUnlock(m_device, &data); return m_deviceFuncs.pfnUnlock(m_device, &data);
} }
m_subResources[data.SubResourceIndex].isLocked = false; m_subResources[data.SubResourceIndex].isLocked = false;
@ -88,7 +89,7 @@ namespace D3dDdi
D3DDDIARG_UNLOCK data = {}; D3DDDIARG_UNLOCK data = {};
data.hResource = m_resource; data.hResource = m_resource;
data.SubResourceIndex = subResourceIndex; data.SubResourceIndex = subResourceIndex;
D3dDdi::DeviceFuncs::s_origVtables.at(m_device).pfnUnlock(m_device, &data); m_deviceFuncs.pfnUnlock(m_device, &data);
subResource.surfacePtr = nullptr; subResource.surfacePtr = nullptr;
subResource.pitch = 0; subResource.pitch = 0;

4
DDrawCompat/D3dDdi/RenderTargetResource.h
View File

@ -11,7 +11,8 @@ namespace D3dDdi
class RenderTargetResource class RenderTargetResource
{ {
public: public:
RenderTargetResource(HANDLE device, HANDLE resource, D3DDDIFORMAT format, UINT surfaceCount); RenderTargetResource(const D3DDDI_DEVICEFUNCS& deviceFuncs,
HANDLE device, HANDLE resource, D3DDDIFORMAT format, UINT surfaceCount);
HRESULT lock(D3DDDIARG_LOCK& data); HRESULT lock(D3DDDIARG_LOCK& data);
HRESULT unlock(const D3DDDIARG_UNLOCK& data); HRESULT unlock(const D3DDDIARG_UNLOCK& data);
@ -33,6 +34,7 @@ namespace D3dDdi
void prepareSubResourceForRendering(UINT subResourceIndex); void prepareSubResourceForRendering(UINT subResourceIndex);
const D3DDDI_DEVICEFUNCS& m_deviceFuncs;
HANDLE m_device; HANDLE m_device;
HANDLE m_resource; HANDLE m_resource;
UINT m_bytesPerPixel; UINT m_bytesPerPixel;