日期:2014-05-18  浏览次数:20588 次

SQL中删除表内的重复数据
表是用于保存修改的历史记录的, 表的结构如下:
TABLE [dbo].[Upd_Record](
[ID] [decimal](18, 0) IDENTITY(1,1) NOT NULL, --标识列
[Table_nm] [varchar](50) NOT NULL,--外联表名
[Table_Cell] [varchar](50) NOT NULL,--单元格名称
[Row_ID] [decimal](18, 0) NOT NULL,--外联主键
[Cell_Value] [decimal](18, 4) NULL,--值 ,主要是这个重复
[upd_date] [datetime] NOT NULL,--修改日期
[upd_id] [decimal](18, 0) NULL--修改人

数据例子如下:
id Table_nm Table_Cell Row_ID Cell_Value 
1 bxsphz_sub clf 327 564.0000 2012-03-23 14:34:46.000 4195
2 bxsphz_sub bxywf 327 495.0000 2012-03-23 14:34:46.000 4195
3 bxsphz_sub tmoney 327 0.0000 2012-03-23 14:34:46.000 4195
4 bxsphz_sub clf 327 564.0000 2012-03-25 14:21:35.000 2121
5 bxsphz_sub bxywf 327 495.0000 2012-03-25 14:21:35.000 2121
6 bxsphz_sub tmoney 327 0.0000 2012-03-25 14:21:35.000 2121

需求如下:
如果有修改的数据,保留;
如果2次修改没有改变过数据, 则只保留最早的,如上面的数据 应保留123,删除456;
由于修改日期基本上是越来越大的,所以按照ID大小来删除应该就可以了, 于是LZ写了如下的 sql语句:

 select * from Upd_Record 
 where ID not in( 
 select g.ID 
 from Upd_Record g 
 where exists ( 
 select f.ID 
from Upd_Record f 
where f.ID < g.ID and not exists (
 select Table_nm, Table_Cell, Row_ID, Cell_Value
 from Upd_Record 
 where ID = f.ID or ID = g.ID
 group by Table_nm, Table_Cell, Row_ID, Cell_Value
having count(*) =1)))
go

用临时表测试的时候,感觉这个应该是可行的,但是复制到数据表中, 由于数据有 45W多条,卡住了10来分钟仍然在查询中,无法得到结果。
因此特来求教, 有没有一些好的办法来处理 删除重复的数据这样的需求?尤其是大批量的时候


------解决方案--------------------
SQL code

删除重复记录有大小关系时,保留大或小其中一个记录


--> --> (Roy)生成測試數據

if not object_id('Tempdb..#T') is null
    drop table #T
Go
Create table #T([ID] int,[Name] nvarchar(1),[Memo] nvarchar(2))
Insert #T
select 1,N'A',N'A1' union all
select 2,N'A',N'A2' union all
select 3,N'A',N'A3' union all
select 4,N'B',N'B1' union all
select 5,N'B',N'B2'
Go

--I、Name相同ID最小的记录(推荐用1,2,3),保留最小一条
方法1:
delete a from #T a where  exists(select 1 from #T where Name=a.Name and ID<a.ID)

方法2:
delete a  from #T a left join (select min(ID)ID,Name from #T group by Name) b on a.Name=b.Name and a.ID=b.ID where b.Id is null

方法3:
delete a from #T a where ID not in (select min(ID) from #T where Name=a.Name)

方法4(注:ID为唯一时可用):
delete a from #T a where ID not in(select min(ID)from #T group by Name)

方法5:
delete a from #T a where (select count(1) from #T where Name=a.Name and ID<a.ID)>0

方法6:
delete a from #T a where ID<>(select top 1 ID from #T where Name=a.name order by ID)

方法7:
delete a from #T a where ID>any(select ID from #T where Name=a.Name)



select * from #T

生成结果:
/*
ID          Name Memo
----------- ---- ----
1           A    A1
4           B    B1

(2 行受影响)
*/


--II、Name相同ID保留最大的一条记录:

方法1:
delete a from #T a where  exists(select 1 from #T where Name=a.Name and ID>a.ID)

方法2:
delete a  from #T a left join (select max(ID)ID,Name from #T group by Name) b on a.Name=b.Name and a.ID=b.ID where b.Id is null

方法3:
delete a from #T a where ID not in (select max(ID) from #T where Name=a.Name)

方法4(注:ID为唯一时可用):
delete a from #T a where ID not in(select max(ID)from #T group by Name)

方法5:
delete a from #T a where (select count(1) from #T where Name=a.Name and ID>a.ID)>0

方法6:
delete a from #T a where ID<>(select top 1 ID from #T where Name=a.name order by ID desc)

方法7:
delete a from #T a where ID<any(select ID from #T where Name=a.Name)


select * from #T
/*
ID          Name Memo
----------- ---- ----
3           A    A3
5           B    B2

(2 行受影响)
*/





--3、删除重复记录没有大小关系时,处理重复值


--> --> (Roy)生成測試數據
 
if not object_id('Tempdb..#T') is null
    drop table #T
Go
Create table #T([Num] int,[Name] nvarchar(1))
Insert #T
select 1,N'A' union all
select 1,N'A' union al