Try this, give your scripts unique keys and let the framework wrap it in script tags for you.
string Result=rptColumnBAL.CheckDuplicateColumnsBAL(calcDetail);
if (Result == calcDetail.DisplayColumnName)
{
Page.ClientScript.RegisterClientScriptBlock(this.GetType(), "alertInvalidColName", "alert('The column name you have chosen is already in use. Please enter a different name');", true);
}
else if (Result == "")
{
Session["CalcFormula"] = calcDetail.calculationFormula;
Session["DisplayColumnName"] = calcDetail.DisplayColumnName;
Session["UseDividendIfZero"] = (cbDividend.Checked == true ? true : false);
Page.ClientScript.RegisterClientScriptBlock(this.GetType(), "CloseWin", "close();", true);
}
else
{
Page.ClientScript.RegisterClientScriptBlock(this.GetType(), "alertInvalidFormula", "alert('The formula you created already exists under the column name: " + Result + "');", true);
};
Also you had single quotes all over the place so chances are your script was being rendered but just causing errors when it ran.