Solved the problem by using HitTestInfo to first check the row.
public bool IsMouseInButtonBoundary(Point point)
{
DataGridView.HitTestInfo ht = m_GridView.HitTest(point.X, point.Y);
if (ht.RowIndex == RowIndex)
{
if (point.X >= X && point.X <= Right)
{
if (point.Y >= Y && point.Y <= Bottom)
{
return true;
}
}
}
return false;
}