I think the issue can with your client id OR option variable. So let try both. if you are using framework4.0 then try to set text box property as ClientIDMode='Static' and use direct textbox id instead of id.
like: $("#yourtextboxid")
Second observation with your option variable try to use direct Ajax call like:
<script type="text/javascript">
var txtsearch = $("#<%=txtSearch.ClientID%>");
$(document).ready(function () {
$("#<%=txtSearch.ClientID%>").keyup(function (e) {
var option = $.ajax({
type: "POST",
url: "searchsmall.aspx/SmSearch",
data:"{}",
contentType: "application/json; charset=utf-8",
dataType: "json",
success: function (msg) {
alert(msg);
}
});
});
});
</script>
Let me know if still not working