加快SQL Server交叉应用以获取聚合数据

在SQL Server中，我试图组合一个查询，该查询获取一行，并包含该行前两小时窗口中的聚合数据以及该行后一小时窗口中的聚合数据。我怎样才能让它跑得更快

行具有毫秒精度的时间戳，并且间隔不均匀。我在这个表中有超过5000万行，查询似乎没有完成。很多地方都有索引，但它们似乎没有帮助。我也在考虑使用窗口函数，但我不确定是否可能有一个行分布不均匀的滑动窗口。另外，对于未来的一个小时窗口，我不确定如何使用SQL窗口实现这一点

框是一个字符串，具有10个唯一值。 进程是一个字符串，具有30个唯一值。 平均持续时间为200ms。 错误占数据的比例不到0.1%。 5000万行描述了一年的数据

select 
c1.start_time,
c1.end_time,
c1.box,
c1.process,
datediff(ms,c1.start_time,c1.end_time) as duration_ms,
datepart(dw,c1.start_time) as day_of_week,
datepart(hour,c1.start_time) as hour_of_day,
c3.*,
c5.*
from metrics_table c1
cross apply
(select 
    avg(cast(datediff(ms,c2.start_time,c2.end_time) as numeric)) as avg_ms,
    count(1) as num_process_total,
    count(distinct process) as num_process_unique,
    count(distinct box) as num_box_unique
    from metrics_table c2
    where datediff(minute,c2.start_time,c1.start_time) <= 120
    and c1.start_time> c2.start_time
    and c2.error_code = 0
) c3
cross apply
(select
    avg(case when datediff(ms,c4.start_time,c4.end_time)>1000 then 1.0 else 0.0 end) as percent_over_thresh
    from metrics_table c4
    where datediff(hour,c1.start_time,c4.start_time) <= 1
    and c4.start_time> c1.start_time
    and c4.error_code= 0
) c5
where
c1.error_code= 0

编辑

版本：SQLAzure 12.0

添加执行计划：

---

以下应该是朝着正确方向迈出的一步。。。 注意：c2.start_time和c4.start_time不再包装在DATEDIFF函数中，因此它们是可分配的

SELECT
    c1.start_time,
    c1.end_time,
    c1.box,
    c1.process,
    DATEDIFF(ms, c1.start_time, c1.end_time) AS duration_ms,
    DATEPART(dw, c1.start_time) AS day_of_week,
    DATEPART(HOUR, c1.start_time) AS hour_of_day,
    --c3.*,
    avg_ms = CASE WHEN 
    c5.*
FROM
    dbo.metrics_table c1
    CROSS APPLY (
                SELECT
                    AVG(CAST(DATEDIFF(ms, c2.start_time, c2.end_time) AS NUMERIC)) AS avg_ms,
                    COUNT(1) AS num_process_total,
                    COUNT(DISTINCT process) AS num_process_unique,
                    COUNT(DISTINCT box) AS num_box_unique
                FROM
                    dbo.metrics_table c2
                WHERE
                    --DATEDIFF(minute,c2.start_time,c1.start_time) <= 120
                    c2.start_time <= DATEADD(MINUTE, -120, c1.start_time)
                    --and c1.start_time> c2.start_time
                    AND c2.error_code = 0
                ) c3
    CROSS APPLY (
                SELECT
                    AVG(CASE WHEN DATEDIFF(ms, c4.start_time, c4.end_time) > 1000 THEN 1.0 ELSE 0.0 END
                    ) AS percent_over_thresh
                FROM
                    dbo.metrics_table c4
                WHERE
                    --DATEDIFF(HOUR, c1.start_time, c4.start_time) <= 1
                    c4.start_time >= DATEADD(HOUR, 1, c1.start_time)
                    --and c4.start_time> c1.start_time
                    AND c4.error_code = 0
                ) c5
WHERE
    c1.error_code = 0;

我使用了中间，并在我的简单试验台上获得了良好的性能。我还使用了columnstore，因为5000万条记录是DW卷：

CREATE TABLE dbo.metrics_table (
    rowId       INT IDENTITY,
    start_time  DATETIME NOT NULL,
    end_time    DATETIME NOT NULL,
    box         VARCHAR(10) NOT NULL,
    process     VARCHAR(10) NOT NULL,
    error_code  INT NOT NULL
);


-- Add records
;WITH cte AS (
SELECT TOP 3334 ROW_NUMBER() OVER ( ORDER BY ( SELECT 1 ) ) rn
FROM sys.columns c1
    CROSS JOIN sys.columns c2
    CROSS JOIN sys.columns c3
)
INSERT INTO dbo.metrics_table ( start_time, end_time, box, process, error_code )
SELECT
    DATEADD( ms, rn, DATEADD( day, rn % 365, '1 Jan 2017' ) ) AS start_time,
    DATEADD( ms, rn % 409, DATEADD( ms, rn, DATEADD( day, rn % 365, '1 Jan 2017' ) ) ) AS end_time,
    'box' + CAST( boxes.box AS VARCHAR(10) ) box,
    'process' + CAST( boxes.box AS VARCHAR(10) ) process,
    ABS( CAST( rn % 3000 AS BIT ) -1 ) error_code
FROM cte c
    CROSS JOIN ( SELECT TOP 10 rn FROM cte ) AS boxes(box)
    CROSS JOIN ( SELECT TOP 30 rn FROM cte ) AS processes(process);


-- Create normal clustered index to order the data
CREATE CLUSTERED INDEX cci_metrics_table ON dbo.metrics_table ( start_time, end_time, box, process );
--CREATE CLUSTERED INDEX cci_metrics_table ON dbo.metrics_table ( box, process, start_time, end_time );

-- Convert to columnstore
CREATE CLUSTERED COLUMNSTORE INDEX cci_metrics_table ON dbo.metrics_table WITH ( MAXDOP = 1, DROP_EXISTING = ON );



IF OBJECT_ID('tempdb..#tmp1' ) IS NOT NULL DROP TABLE #tmp1

-- two hour window before, 1 hour window after
SELECT
    c1.start_time,
    c1.end_time,
    c1.box,
    c1.process,
    DATEDIFF( ms, c1.start_time, c1.end_time ) AS duration_ms,
    DATEPART( dw, c1.start_time ) AS day_of_week,
    DATEPART( hour, c1.start_time ) AS hour_of_day,
    c2.xavg,
    c2.num_process_total,
    c2.num_process_unique,
    c2.num_box_unique,
    c3.percent_over_thresh

INTO #tmp1

FROM dbo.metrics_table c1
    CROSS APPLY
        (
        SELECT
            COUNT(1) AS num_process_total, 
            AVG( CAST( DATEDIFF( ms, start_time, end_time ) AS NUMERIC ) ) xavg,
            COUNT( DISTINCT process ) num_process_unique,
            COUNT( DISTINCT box ) num_box_unique
        FROM dbo.metrics_table c2
        WHERE c2.error_code = 0
          AND c2.start_time Between DATEADD( minute, -120, c1.start_time ) And c1.start_time
          AND c1.start_time > c2.start_time
        ) c2

    CROSS APPLY
        (
        SELECT
            AVG( CASE WHEN DATEDIFF( ms, c4.start_time, c4.end_time ) > 1000 THEN 1.0 ELSE 0.0 END ) percent_over_thresh
        FROM dbo.metrics_table c4
        WHERE c4.error_code = 0
          AND c4.start_time Between c1.start_time And DATEADD( minute, 60, c1.start_time )
          AND c4.start_time > c1.start_time
        ) c3

WHERE error_code = 0

---

如果性能问题不是因为where谓词，我会感到惊讶。where子句中有函数，这意味着您必须为每一行计算datediff。在这种情况下，你要做两次。这意味着您正在执行大约1亿次datediff计算。@Hogan我曾尝试过窗口化，但我没有看到一种方法允许我从一个时间点开始计算-2个小时，如果数据点不是以偶数间隔收集的话。这意味着从一行到下一行的差异可能是几毫秒，可能是几秒钟，可能是几分钟。这在SQL Server实现中是不可能的。间隔之间没有范围您必须进行一些预聚合以保证每分钟一行等。但是COUNTDISTINCT。。。您确实需要去掉datediff，这不是因为执行的次数，而是因为它使查询无法组织。您需要获取嵌套的循环联接，以便对errorcode执行索引查找，然后获取starttime范围，但查询速度可能仍然很慢。还有datediffhour，c1.开始时间，c4.开始时间真的是你想要的吗？00:01和01:59的datediff仍然为1，因为它计算跨越的小时边界。

CREATE TABLE dbo.metrics_table (
    rowId       INT IDENTITY,
    start_time  DATETIME NOT NULL,
    end_time    DATETIME NOT NULL,
    box         VARCHAR(10) NOT NULL,
    process     VARCHAR(10) NOT NULL,
    error_code  INT NOT NULL
);


-- Add records
;WITH cte AS (
SELECT TOP 3334 ROW_NUMBER() OVER ( ORDER BY ( SELECT 1 ) ) rn
FROM sys.columns c1
    CROSS JOIN sys.columns c2
    CROSS JOIN sys.columns c3
)
INSERT INTO dbo.metrics_table ( start_time, end_time, box, process, error_code )
SELECT
    DATEADD( ms, rn, DATEADD( day, rn % 365, '1 Jan 2017' ) ) AS start_time,
    DATEADD( ms, rn % 409, DATEADD( ms, rn, DATEADD( day, rn % 365, '1 Jan 2017' ) ) ) AS end_time,
    'box' + CAST( boxes.box AS VARCHAR(10) ) box,
    'process' + CAST( boxes.box AS VARCHAR(10) ) process,
    ABS( CAST( rn % 3000 AS BIT ) -1 ) error_code
FROM cte c
    CROSS JOIN ( SELECT TOP 10 rn FROM cte ) AS boxes(box)
    CROSS JOIN ( SELECT TOP 30 rn FROM cte ) AS processes(process);


-- Create normal clustered index to order the data
CREATE CLUSTERED INDEX cci_metrics_table ON dbo.metrics_table ( start_time, end_time, box, process );
--CREATE CLUSTERED INDEX cci_metrics_table ON dbo.metrics_table ( box, process, start_time, end_time );

-- Convert to columnstore
CREATE CLUSTERED COLUMNSTORE INDEX cci_metrics_table ON dbo.metrics_table WITH ( MAXDOP = 1, DROP_EXISTING = ON );



IF OBJECT_ID('tempdb..#tmp1' ) IS NOT NULL DROP TABLE #tmp1

-- two hour window before, 1 hour window after
SELECT
    c1.start_time,
    c1.end_time,
    c1.box,
    c1.process,
    DATEDIFF( ms, c1.start_time, c1.end_time ) AS duration_ms,
    DATEPART( dw, c1.start_time ) AS day_of_week,
    DATEPART( hour, c1.start_time ) AS hour_of_day,
    c2.xavg,
    c2.num_process_total,
    c2.num_process_unique,
    c2.num_box_unique,
    c3.percent_over_thresh

INTO #tmp1

FROM dbo.metrics_table c1
    CROSS APPLY
        (
        SELECT
            COUNT(1) AS num_process_total, 
            AVG( CAST( DATEDIFF( ms, start_time, end_time ) AS NUMERIC ) ) xavg,
            COUNT( DISTINCT process ) num_process_unique,
            COUNT( DISTINCT box ) num_box_unique
        FROM dbo.metrics_table c2
        WHERE c2.error_code = 0
          AND c2.start_time Between DATEADD( minute, -120, c1.start_time ) And c1.start_time
          AND c1.start_time > c2.start_time
        ) c2

    CROSS APPLY
        (
        SELECT
            AVG( CASE WHEN DATEDIFF( ms, c4.start_time, c4.end_time ) > 1000 THEN 1.0 ELSE 0.0 END ) percent_over_thresh
        FROM dbo.metrics_table c4
        WHERE c4.error_code = 0
          AND c4.start_time Between c1.start_time And DATEADD( minute, 60, c1.start_time )
          AND c4.start_time > c1.start_time
        ) c3

WHERE error_code = 0