You may need to tweak exactly what you are doing with the permissions in order to remove existing permissions first, but the main problem is that your
User
information is rubbish - that's basically the PC name, rather than a user identifier.
Try this instead:
string dest = @"D:\Directory Test";
DirectoryInfo mydirInfo = new DirectoryInfo(dest);
DirectorySecurity mydirSec = mydirInfo.GetAccessControl();
SecurityIdentifier User = new SecurityIdentifier(WellKnownSidType.WorldSid, null);
mydirSec.AddAccessRule(new FileSystemAccessRule(
User,
FileSystemRights.Read,
InheritanceFlags.ObjectInherit,
PropagationFlags.InheritOnly,
AccessControlType.Allow));
mydirInfo.SetAccessControl(mydirSec);