Try this
<html xmlns="http://www.w3.org/1999/xhtml">
<head id="Head1" runat="server">
<title></title>
<script type="text/javascript">
var OpenPopup = function () {
var value = window.showModalDialog("POPUPPage.aspx", 'popup', "dialogHeight:200; dialogWidth:200; center:yes;toolbar: false;status: 0;scroll:0;unadorned:0;help:no");
document.getElementById('txtValues').value = value;
}
</script>
</head>
<body>
<form id="frm" runat="server">
<asp:TextBox ID="txtValues" runat="server"></asp:TextBox>
<asp:Button ID="btnPopUP" OnClientClick="OpenPopup(); return false;" runat="server"
Text="LookUP" />
</form>
</body>
</html>
POPUP Page:
<%@ Page Language="C#" AutoEventWireup="true" CodeBehind="POPUPPage.aspx.cs" Inherits="POC.POPUPPage" %>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head runat="server">
<title></title>
<script type="text/javascript">
var closepopup = function () {
var temp = '';
var radiolist = document.getElementById('<%= RadioButtonList1.ClientID %>');
var radio = radiolist.getElementsByTagName("input");
for (var x = 0; x < radio.length; x++) {
if (radio[x].type === "radio" && radio[x].checked) {
temp = radio[x].value;
break;
}
}
window.returnValue = temp;
window.close()
}
</script>
</head>
<body>
<form id="form1" runat="server">
<div>
<asp:RadioButtonList ID="RadioButtonList1" runat="server">
<asp:ListItem Text="One" />
<asp:ListItem Text="two" />
<asp:ListItem Text="three" />
<asp:ListItem Text="four" />
<asp:ListItem Text="five" />
</asp:RadioButtonList>
<asp:Button ID="btnClose" runat="server" OnClientClick="closepopup(); return false;"
Text="Select & Close" />
</div>
</form>
</body>
</html>