How to remove duplicate rows in SQL Server 2005 when no key is present

IF OBJECT_ID( 'tempdb..#TestTable' ) IS NOT NULL
    BEGIN
    DROP TABLE #TestTable
    END

CREATE TABLE #TestTable
    (
    Column1 varchar(1),
    Column2 int
    )

INSERT INTO #TestTable VALUES ('A', 1);
INSERT INTO #TestTable VALUES ('A', 1); -- duplicate
INSERT INTO #TestTable VALUES ('A', 1); -- duplicate
INSERT INTO #TestTable VALUES ('A', 2);
INSERT INTO #TestTable VALUES ('B', 1);
INSERT INTO #TestTable VALUES ('B', 2);
INSERT INTO #TestTable VALUES ('B', 2); -- duplicate
INSERT INTO #TestTable VALUES ('C', 1);
INSERT INTO #TestTable VALUES ('C', 2);

SELECT  *
FROM    #TestTable
ORDER
BY      Column1,
        Column2

;WITH dup AS
    (
    SELECT  [DuplicateID] = ROW_NUMBER() OVER(PARTITION BY Column1,Column2 ORDER BY Column1,Column2 )
    --,*
    FROM    #TestTable
    )
DELETE FROM dup
WHERE   [DuplicateID] > 1

SELECT  *
FROM    #TestTable
ORDER
BY      Column1,
        Column2