Sql server 2008 将SQL Server 2008查询分解为批处理
我正试图准备一些数据供第三方删除,不幸的是,他们只能批量处理2000条记录中的数据。我有10万条记录,可能需要多次分割和导出这些数据,所以我想以某种方式自动化这个过程Sql server 2008 将SQL Server 2008查询分解为批处理,sql-server-2008,batch-processing,Sql Server 2008,Batch Processing,我正试图准备一些数据供第三方删除,不幸的是,他们只能批量处理2000条记录中的数据。我有10万条记录,可能需要多次分割和导出这些数据,所以我想以某种方式自动化这个过程 使用SQLServer2008是否有一种相当简单的方法可以做到这一点?我并没有运行复杂的查询—它离按PKID从某个表顺序选择PKID不太远—虽然我可能可以使用光标来执行此操作,但我想知道是否有更好的方法。我认为您可以利用使用ROW\u NUMBER,然后使用BETWEEN来指定您喜欢的行范围。或者,如果您知道不存在漏洞,或者不关心
使用SQLServer2008是否有一种相当简单的方法可以做到这一点?我并没有运行复杂的查询—它离按PKID从某个表顺序选择PKID不太远—虽然我可能可以使用光标来执行此操作,但我想知道是否有更好的方法。我认为您可以利用使用ROW\u NUMBER,然后使用BETWEEN来指定您喜欢的行范围。或者,如果您知道不存在漏洞,或者不关心漏洞,则可以使用PKID e、 g
这通常用于分页结果。4GuysFromRolla有一个很好的您可以在while@@ROWCOUNT循环中计算出范围,以确定所需的行。它可能比ROW_NUMBER更好,ROW_NUMBER必须从一开始就保持编号
declare @startid int
declare @endid int
-- get one range, these are efficient as they go over the PKID key by range
select top(1) @startid = pkid from sometable order by pkid -- 1 key visited
select top(2000) @endid = pkid from sometable order by pkid -- 2000 keys visited
-- note: top 2000 may end up with the 514th id if that is the last one
while @@ROWCOUNT > 0
begin
insert otherdb.dbo.backupcopy
select * from sometable
where pkid between @startid and @endid
select top(1) @startid = pkid from sometable
WHERE pkid > @endid -- binary locate
order by pkid
select top(2000) @endid = pkid from sometable
WHERE pkid > @endid -- binary locate, then forward range lookup, max 2000 keys
order by pkid
end
我最终使用了cyberkiwi和Adam提供的方法的组合。我不需要使用ROW_NUMBER,因为我在表数据类型中使用了标识列 这是我使用的代码的一个修订版本,它工作起来很有魅力。再次感谢大家的帮助
use Testing
GO
SET NOCOUNT ON
declare
@now datetime = GETDATE(),
@batchsize int = 2000,
@bcpTargetDir varchar(500) = '\\SomeServer\Upload\',
@csvQueryServer varchar(500) = '.\SQLExpress',
@rowcount integer,
@nowstring varchar(100),
@batch_id int,
@startid int,
@endid int,
@oidCSV varchar(max),
@csvQuery varchar(max),
@bcpFilename varchar(200),
@bcpQuery varchar(1000)
declare @tblBatchRanges table (
batch_id integer NOT NULL IDENTITY(1,1) PRIMARY KEY,
oid_start integer NOT NULL,
oid_end integer NOT NULL,
csvQuery varchar(max)
)
-- Create a unique timestamp-based string, which will be used to name the exported files.
select @nowstring = CONVERT(varchar, @now, 112) + '-' + REPLACE(CONVERT(varchar, @now, 114), ':', '')
--
select top(1) @startid = oid from Testing..MyObjectIds order by oid
select top(@batchsize) @endid = oid from Testing..MyObjectIds order by oid
select @rowcount = @@ROWCOUNT
while (@rowcount > 0) begin
-- Create a CSV of all object IDs in the batch, using the STUFF() function (http://goo.gl/EyE8L).
select @csvQuery = 'select stuff((select distinct '','' + CAST(oid as varchar) from Testing..MyObjectIds where oid between ' + CAST(@startid as varchar) + ' and ' + CAST(@endid as varchar) + ' order by '','' + CAST(oid as varchar) for xml path('''')),1,1,'''')'
-- Log the info and get the batch ID.
insert into @tblBatchRanges (oid_start, oid_end, csvQuery)
values (@startid, @endid, @oidCSV, @csvQuery)
select @batch_id = @@IDENTITY
-- Advance @startid and @endid so that they point to the next batch
select top(1) @startid = oid
from Testing..MyObjectIds
where oid > @endid
order by oid
select top(@batchsize) @endid = oid
from Testing..MyObjectIds
where oid > @endid
order by oid
select @rowcount = @@ROWCOUNT
-- Export the current batch to a file.
select @bcpFilename = 'MyExport-' + @nowstring + '-' + cast(@batch_id as varchar) + '.txt'
select @bcpQuery = 'bcp "' + @csvQuery + '" QUERYOUT "' + @bcpTargetDir + @bcpFilename + '" -S ' + @csvQueryServer + ' -T -c'
exec master..xp_cmdshell @bcpquery
end
SET NOCOUNT OFF
--Check all of the logged info.
select oid_start, oid_end, csvQuery from @tblBatchRanges
是的,在没有更优雅的解决方案的情况下,这就是我要走的方向。但是,我没有太多处理行数的经验,因此您的示例仍然非常有用:行数在其范围内越远,是否会逐渐变慢?Esp接近100k记录的结尾非常有趣-它直接导出到文件这一事实特别好。我还没有机会试一下,但今天下午我要试一试。太好了!这看起来很棒-我可以尝试将它与Adam的帖子结合起来,将输出直接导出到文本文件中。然而,我不理解下面的评论:注意:Top2000可能会以第514个id结束,如果这是最后一个id的话。你能解释一下吗?即使你要的是top2000,如果只剩下514条记录,那么最后一条记录就是514条。多行选择中的赋值是通过赋值给最后一行的值来工作的,因此@endid=pkid被赋值给最后一行的第514行的id,如果没有2000行。哦,我明白了-你的意思是,这就是你处理最后一批不完全包含2000行的情况的方式,对吗?@AspNyc-完全正确。第一批也可能是最后一批。
declare @startid int
declare @endid int
-- get one range, these are efficient as they go over the PKID key by range
select top(1) @startid = pkid from sometable order by pkid -- 1 key visited
select top(2000) @endid = pkid from sometable order by pkid -- 2000 keys visited
-- note: top 2000 may end up with the 514th id if that is the last one
while @@ROWCOUNT > 0
begin
insert otherdb.dbo.backupcopy
select * from sometable
where pkid between @startid and @endid
select top(1) @startid = pkid from sometable
WHERE pkid > @endid -- binary locate
order by pkid
select top(2000) @endid = pkid from sometable
WHERE pkid > @endid -- binary locate, then forward range lookup, max 2000 keys
order by pkid
end
use Testing
GO
SET NOCOUNT ON
declare
@now datetime = GETDATE(),
@batchsize int = 2000,
@bcpTargetDir varchar(500) = '\\SomeServer\Upload\',
@csvQueryServer varchar(500) = '.\SQLExpress',
@rowcount integer,
@nowstring varchar(100),
@batch_id int,
@startid int,
@endid int,
@oidCSV varchar(max),
@csvQuery varchar(max),
@bcpFilename varchar(200),
@bcpQuery varchar(1000)
declare @tblBatchRanges table (
batch_id integer NOT NULL IDENTITY(1,1) PRIMARY KEY,
oid_start integer NOT NULL,
oid_end integer NOT NULL,
csvQuery varchar(max)
)
-- Create a unique timestamp-based string, which will be used to name the exported files.
select @nowstring = CONVERT(varchar, @now, 112) + '-' + REPLACE(CONVERT(varchar, @now, 114), ':', '')
--
select top(1) @startid = oid from Testing..MyObjectIds order by oid
select top(@batchsize) @endid = oid from Testing..MyObjectIds order by oid
select @rowcount = @@ROWCOUNT
while (@rowcount > 0) begin
-- Create a CSV of all object IDs in the batch, using the STUFF() function (http://goo.gl/EyE8L).
select @csvQuery = 'select stuff((select distinct '','' + CAST(oid as varchar) from Testing..MyObjectIds where oid between ' + CAST(@startid as varchar) + ' and ' + CAST(@endid as varchar) + ' order by '','' + CAST(oid as varchar) for xml path('''')),1,1,'''')'
-- Log the info and get the batch ID.
insert into @tblBatchRanges (oid_start, oid_end, csvQuery)
values (@startid, @endid, @oidCSV, @csvQuery)
select @batch_id = @@IDENTITY
-- Advance @startid and @endid so that they point to the next batch
select top(1) @startid = oid
from Testing..MyObjectIds
where oid > @endid
order by oid
select top(@batchsize) @endid = oid
from Testing..MyObjectIds
where oid > @endid
order by oid
select @rowcount = @@ROWCOUNT
-- Export the current batch to a file.
select @bcpFilename = 'MyExport-' + @nowstring + '-' + cast(@batch_id as varchar) + '.txt'
select @bcpQuery = 'bcp "' + @csvQuery + '" QUERYOUT "' + @bcpTargetDir + @bcpFilename + '" -S ' + @csvQueryServer + ' -T -c'
exec master..xp_cmdshell @bcpquery
end
SET NOCOUNT OFF
--Check all of the logged info.
select oid_start, oid_end, csvQuery from @tblBatchRanges