You have to convert two hexadecimal characters at time, e.g.
string s = "ed07";
vector <unsigned char> binary;
for (string::size_type i=0; i<s.length(); i+=2)
{
unsigned char b = (unsigned char) strtoul(s.substr(i,2).c_str(), NULL, 16);
binary.push_back(b);
}