ATL / WTL / STL

Stuart Dootson wrote:

Before you think about that - are you using CWindow objects? ATL 3.0 (&7.1, IIRC) relied on constructing code thunks in data memory to perform dispatching of window messages to C++ objects. This is precisely the sort of code that DEP is intended to protect against.

If you believe your DLL is OK - what's to stop you using the editbin.exe from VS2005? To make it easier, copy it into your solution's root directory and use it from there.

// SetNXCompat.cpp : Defines the entry point for the console application.
//
 
#include "stdafx.h"
#include <windows.h>
#include <tchar.h>
#include <Imagehlp.h>
#include <shlwapi.h>
#include <malloc.h>
#include <iostream>
#include <string>
#pragma comment(lib, "Imagehlp.lib")
#pragma comment(lib, "shlwapi.lib")
using namespace std;
 
// In case you're using an old SDK that doesn't include the flag.
#ifndef IMAGE_DLLCHARACTERISTICS_NX_COMPAT
#	define IMAGE_DLLCHARACTERISTICS_NX_COMPAT 0x0100
#endif
 
// We'll be nice and support both ANSI and UNICODE.
#ifdef UNICODE
#	define tcout wcout
#	define tcerr wcerr
	typedef wstring tstring;
#else
#	define tcout cout
#	define tcerr cerr
	typedef string tstring;
#endif
 
void PrintUsage()
{
	tcout << _T("Usage:") << endl;
	tcout << _T("\tSetNXCompat <file>") << endl;
}
 
bool Worker(LPCTSTR pDir, LPCTSTR pFile)
{
	// Load the image.
	LOADED_IMAGE li;
	BOOL ok = MapAndLoad(
			const_cast<LPTSTR>(pFile),	// PSTR ImageName
			const_cast<LPTSTR>(pDir),	// PSTR DllPath
			&li,				// PLOADED_IMAGE LoadedImage
			FALSE,				// BOOL DotDll
			FALSE				// BOOL ReadOnly
			);
	if (!ok)
	{
		return false;
	}
 
	// Set the flag.
	IMAGE_NT_HEADERS *pHeaders = li.FileHeader;
	pHeaders->OptionalHeader.DllCharacteristics |= IMAGE_DLLCHARACTERISTICS_NX_COMPAT;
 
	// We'll go the extra mile and set the checksum.
	DWORD org_sum;
	DWORD new_sum;
	IMAGE_NT_HEADERS *pNT = CheckSumMappedFile(
					li.MappedAddress,	// PVOID BaseAddress
					li.SizeOfImage,		// DWORD FileLength
					&org_sum,		// PDWORD HeaderSum
					&new_sum		// PDWORD CheckSum
					);
	if (pNT != NULL)
	{
		pNT->OptionalHeader.CheckSum = new_sum;
	}
	else
	{
		tcerr << "WARNING: Failed to update checksum." << endl;
	}
 
	// Were done!
	ok = UnMapAndLoad(&li);
 
	return (ok) ? true : false;
}
 
int main(int argc, TCHAR* argv[])
{
	if (argc != 2)
	{
		PrintUsage();
		return -1;
	}
 
	// Get the directory.
	tstring dir;
	if (PathIsRelative(argv[1]))
	{
		// Get the current directory.
		DWORD len = GetCurrentDirectory(0, NULL);
		LPTSTR pBuffer = static_cast<LPTSTR>(_alloca((len+1)*sizeof(TCHAR)));
		GetCurrentDirectory(len+1, pBuffer);
		dir = pBuffer;
	}
	else
	{
		dir = argv[1];
		tstring::size_type pos = dir.find_last_of(_T("\\/"));
		if (pos == -1)
		{
			tcerr << "Error in file path!" << endl;
			return -2;
		}
 
		dir.erase(pos);
	}
 
	// Feed the worker.
	if (!Worker(dir.c_str(), argv[1]))
	{
		tcerr << "Failed!" << endl;
		return -3;
	}
 
	return 0;
}