Mongodb 用于复杂$and和$or查询的Mongo索引
我正在尝试使用各种选项来优化查询响应。这个名为nodeData的集合拥有2000万条记录。我的查询命中的是磁盘，而不是索引，并使服务器在其余查询中的运行速度减慢Mongodb 用于复杂$and和$or查询的Mongo索引,mongodb,database-design,mongodb-query,database-performance,Mongodb,Database Design,Mongodb Query,Database Performance,我正在尝试使用各种选项来优化查询响应。这个名为nodeData的集合拥有2000万条记录。我的查询命中的是磁盘，而不是索引，并使服务器在其余查询中的运行速度减慢 db.nodeData.find({ "$or" : [ { "neid" : "40-cb-28-dd-46-e7-d9-c3"} , { "$and" : [ { "hubId" : "87-0c-4e-a6-ff-39-cc-94"} , { "nodeId" : 1}]}] , "capturedAt" : { "$lte" :
db.nodeData.find({ "$or" : [ { "neid" : "40-cb-28-dd-46-e7-d9-c3"} , { "$and" : [ { "hubId" : "87-0c-4e-a6-ff-39-cc-94"} , { "nodeId" : 1}]}] , "capturedAt" : { "$lte" : new ISODate("2017-11-19T05:17:30.000Z") , "$gt" : new ISODate("2017-11-12T05:17:30.000Z")}}).explain("executionStats")
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "wizn_iot.nodeData",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"$or" : [
{
"$and" : [
{
"hubId" : {
"$eq" : "87-0c-4e-a6-ff-39-cc-94"
}
},
{
"nodeId" : {
"$eq" : 1
}
}
]
},
{
"neid" : {
"$eq" : "40-cb-28-dd-46-e7-d9-c3"
}
}
]
},
{
"capturedAt" : {
"$lte" : ISODate("2017-11-19T05:17:30Z")
}
},
{
"capturedAt" : {
"$gt" : ISODate("2017-11-12T05:17:30Z")
}
}
]
},
"winningPlan" : {
"stage" : "KEEP_MUTATIONS",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"capturedAt" : {
"$lte" : ISODate("2017-11-19T05:17:30Z")
}
},
{
"capturedAt" : {
"$gt" : ISODate("2017-11-12T05:17:30Z")
}
}
]
},
"inputStage" : {
"stage" : "OR",
"inputStages" : [
{
"stage" : "IXSCAN",
"keyPattern" : {
"hubId" : 1,
"nodeId" : 1
},
"indexName" : "hubId_1_nodeId_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"hubId" : [
"[\"87-0c-4e-a6-ff-39-cc-94\", \"87-0c-4e-a6-ff-39-cc-94\"]"
],
"nodeId" : [
"[1.0, 1.0]"
]
}
},
{
"stage" : "IXSCAN",
"keyPattern" : {
"neid" : 1,
"hubId" : 1,
"nodeId" : 1,
"capturedAt" : 1
},
"indexName" : "neid_1_hubId_1_nodeId_1_capturedAt_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"neid" : [
"[\"40-cb-28-dd-46-e7-d9-c3\", \"40-cb-28-dd-46-e7-d9-c3\"]"
],
"hubId" : [
"[MinKey, MaxKey]"
],
"nodeId" : [
"[MinKey, MaxKey]"
],
"capturedAt" : [
"[MinKey, MaxKey]"
]
}
}
]
}
}
},
"rejectedPlans" : [
{
"stage" : "KEEP_MUTATIONS",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"capturedAt" : {
"$lte" : ISODate("2017-11-19T05:17:30Z")
}
},
{
"capturedAt" : {
"$gt" : ISODate("2017-11-12T05:17:30Z")
}
}
]
},
"inputStage" : {
"stage" : "OR",
"inputStages" : [
{
"stage" : "IXSCAN",
"keyPattern" : {
"hubId" : 1,
"nodeId" : 1,
"capturedAt" : 1
},
"indexName" : "hubId_1_nodeId_1_capturedAt_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"hubId" : [
"[\"87-0c-4e-a6-ff-39-cc-94\", \"87-0c-4e-a6-ff-39-cc-94\"]"
],
"nodeId" : [
"[1.0, 1.0]"
],
"capturedAt" : [
"[MinKey, MaxKey]"
]
}
},
{
"stage" : "IXSCAN",
"keyPattern" : {
"neid" : 1,
"hubId" : 1,
"nodeId" : 1,
"capturedAt" : 1
},
"indexName" : "neid_1_hubId_1_nodeId_1_capturedAt_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"neid" : [
"[\"40-cb-28-dd-46-e7-d9-c3\", \"40-cb-28-dd-46-e7-d9-c3\"]"
],
"hubId" : [
"[MinKey, MaxKey]"
],
"nodeId" : [
"[MinKey, MaxKey]"
],
"capturedAt" : [
"[MinKey, MaxKey]"
]
}
}
]
}
}
},
{
"stage" : "KEEP_MUTATIONS",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"capturedAt" : {
"$lte" : ISODate("2017-11-19T05:17:30Z")
}
},
{
"capturedAt" : {
"$gt" : ISODate("2017-11-12T05:17:30Z")
}
}
]
},
"inputStage" : {
"stage" : "OR",
"inputStages" : [
{
"stage" : "IXSCAN",
"keyPattern" : {
"hubId" : 1,
"nodeId" : 1,
"neid" : 1,
"capturedAt" : 1
},
"indexName" : "hubId_1_nodeId_1_neid_1_capturedAt_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"hubId" : [
"[\"87-0c-4e-a6-ff-39-cc-94\", \"87-0c-4e-a6-ff-39-cc-94\"]"
],
"nodeId" : [
"[1.0, 1.0]"
],
"neid" : [
"[MinKey, MaxKey]"
],
"capturedAt" : [
"[MinKey, MaxKey]"
]
}
},
{
"stage" : "IXSCAN",
"keyPattern" : {
"neid" : 1,
"hubId" : 1,
"nodeId" : 1,
"capturedAt" : 1
},
"indexName" : "neid_1_hubId_1_nodeId_1_capturedAt_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"neid" : [
"[\"40-cb-28-dd-46-e7-d9-c3\", \"40-cb-28-dd-46-e7-d9-c3\"]"
],
"hubId" : [
"[MinKey, MaxKey]"
],
"nodeId" : [
"[MinKey, MaxKey]"
],
"capturedAt" : [
"[MinKey, MaxKey]"
]
}
}
]
}
}
},
{
"stage" : "KEEP_MUTATIONS",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"capturedAt" : {
"$lte" : ISODate("2017-11-19T05:17:30Z")
}
},
{
"capturedAt" : {
"$gt" : ISODate("2017-11-12T05:17:30Z")
}
}
]
},
"inputStage" : {
"stage" : "OR",
"inputStages" : [
{
"stage" : "FETCH",
"filter" : {
"nodeId" : {
"$eq" : 1
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"hubId" : 1
},
"indexName" : "hubId_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"hubId" : [
"[\"87-0c-4e-a6-ff-39-cc-94\", \"87-0c-4e-a6-ff-39-cc-94\"]"
]
}
}
},
{
"stage" : "IXSCAN",
"keyPattern" : {
"neid" : 1,
"hubId" : 1,
"nodeId" : 1,
"capturedAt" : 1
},
"indexName" : "neid_1_hubId_1_nodeId_1_capturedAt_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"neid" : [
"[\"40-cb-28-dd-46-e7-d9-c3\", \"40-cb-28-dd-46-e7-d9-c3\"]"
],
"hubId" : [
"[MinKey, MaxKey]"
],
"nodeId" : [
"[MinKey, MaxKey]"
],
"capturedAt" : [
"[MinKey, MaxKey]"
]
}
}
]
}
}
},
{
"stage" : "KEEP_MUTATIONS",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"capturedAt" : {
"$lte" : ISODate("2017-11-19T05:17:30Z")
}
},
{
"capturedAt" : {
"$gt" : ISODate("2017-11-12T05:17:30Z")
}
}
]
},
"inputStage" : {
"stage" : "OR",
"inputStages" : [
{
"stage" : "IXSCAN",
"keyPattern" : {
"hubId" : 1,
"nodeId" : 1,
"capturedAt" : 1
},
"indexName" : "hubId_1_nodeId_1_capturedAt_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"hubId" : [
"[\"87-0c-4e-a6-ff-39-cc-94\", \"87-0c-4e-a6-ff-39-cc-94\"]"
],
"nodeId" : [
"[1.0, 1.0]"
],
"capturedAt" : [
"[MinKey, MaxKey]"
]
}
},
{
"stage" : "IXSCAN",
"keyPattern" : {
"neid" : 1
},
"indexName" : "neid_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"neid" : [
"[\"40-cb-28-dd-46-e7-d9-c3\", \"40-cb-28-dd-46-e7-d9-c3\"]"
]
}
}
]
}
}
},
{
"stage" : "KEEP_MUTATIONS",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"capturedAt" : {
"$lte" : ISODate("2017-11-19T05:17:30Z")
}
},
{
"capturedAt" : {
"$gt" : ISODate("2017-11-12T05:17:30Z")
}
}
]
},
"inputStage" : {
"stage" : "OR",
"inputStages" : [
{
"stage" : "IXSCAN",
"keyPattern" : {
"hubId" : 1,
"nodeId" : 1
},
"indexName" : "hubId_1_nodeId_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"hubId" : [
"[\"87-0c-4e-a6-ff-39-cc-94\", \"87-0c-4e-a6-ff-39-cc-94\"]"
],
"nodeId" : [
"[1.0, 1.0]"
]
}
},
{
"stage" : "IXSCAN",
"keyPattern" : {
"neid" : 1
},
"indexName" : "neid_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"neid" : [
"[\"40-cb-28-dd-46-e7-d9-c3\", \"40-cb-28-dd-46-e7-d9-c3\"]"
]
}
}
]
}
}
},
{
"stage" : "KEEP_MUTATIONS",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"capturedAt" : {
"$lte" : ISODate("2017-11-19T05:17:30Z")
}
},
{
"capturedAt" : {
"$gt" : ISODate("2017-11-12T05:17:30Z")
}
}
]
},
"inputStage" : {
"stage" : "OR",
"inputStages" : [
{
"stage" : "IXSCAN",
"keyPattern" : {
"hubId" : 1,
"nodeId" : 1,
"neid" : 1,
"capturedAt" : 1
},
"indexName" : "hubId_1_nodeId_1_neid_1_capturedAt_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"hubId" : [
"[\"87-0c-4e-a6-ff-39-cc-94\", \"87-0c-4e-a6-ff-39-cc-94\"]"
],
"nodeId" : [
"[1.0, 1.0]"
],
"neid" : [
"[MinKey, MaxKey]"
],
"capturedAt" : [
"[MinKey, MaxKey]"
]
}
},
{
"stage" : "IXSCAN",
"keyPattern" : {
"neid" : 1
},
"indexName" : "neid_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"neid" : [
"[\"40-cb-28-dd-46-e7-d9-c3\", \"40-cb-28-dd-46-e7-d9-c3\"]"
]
}
}
]
}
}
},
{
"stage" : "KEEP_MUTATIONS",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"capturedAt" : {
"$lte" : ISODate("2017-11-19T05:17:30Z")
}
},
{
"capturedAt" : {
"$gt" : ISODate("2017-11-12T05:17:30Z")
}
}
]
},
"inputStage" : {
"stage" : "OR",
"inputStages" : [
{
"stage" : "FETCH",
"filter" : {
"nodeId" : {
"$eq" : 1
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"hubId" : 1
},
"indexName" : "hubId_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"hubId" : [
"[\"87-0c-4e-a6-ff-39-cc-94\", \"87-0c-4e-a6-ff-39-cc-94\"]"
]
}
}
},
{
"stage" : "IXSCAN",
"keyPattern" : {
"neid" : 1
},
"indexName" : "neid_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"neid" : [
"[\"40-cb-28-dd-46-e7-d9-c3\", \"40-cb-28-dd-46-e7-d9-c3\"]"
]
}
}
]
}
}
},
{
"stage" : "KEEP_MUTATIONS",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$or" : [
{
"$and" : [
{
"hubId" : {
"$eq" : "87-0c-4e-a6-ff-39-cc-94"
}
},
{
"nodeId" : {
"$eq" : 1
}
}
]
},
{
"neid" : {
"$eq" : "40-cb-28-dd-46-e7-d9-c3"
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"capturedAt" : 1
},
"indexName" : "capturedAt_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"capturedAt" : [
"(new Date(1510463850000), new Date(1511068650000)]"
]
}
}
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1408,
"executionTimeMillis" : 28908,
"totalKeysExamined" : 16478,
"totalDocsExamined" : 16478,
"executionStages" : {
"stage" : "KEEP_MUTATIONS",
"nReturned" : 1408,
"executionTimeMillisEstimate" : 2620,
"works" : 16531,
"advanced" : 1408,
"needTime" : 15071,
"needYield" : 51,
"saveState" : 2114,
"restoreState" : 2114,
"isEOF" : 1,
"invalidates" : 0,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"capturedAt" : {
"$lte" : ISODate("2017-11-19T05:17:30Z")
}
},
{
"capturedAt" : {
"$gt" : ISODate("2017-11-12T05:17:30Z")
}
}
]
},
"nReturned" : 1408,
"executionTimeMillisEstimate" : 2620,
"works" : 16531,
"advanced" : 1408,
"needTime" : 15071,
"needYield" : 51,
"saveState" : 2114,
"restoreState" : 2114,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 16478,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "OR",
"nReturned" : 16478,
"executionTimeMillisEstimate" : 140,
"works" : 16480,
"advanced" : 16478,
"needTime" : 1,
"needYield" : 0,
"saveState" : 2114,
"restoreState" : 2114,
"isEOF" : 1,
"invalidates" : 0,
"dupsTested" : 16478,
"dupsDropped" : 0,
"locsForgotten" : 0,
"inputStages" : [
{
"stage" : "IXSCAN",
"nReturned" : 16478,
"executionTimeMillisEstimate" : 30,
"works" : 16479,
"advanced" : 16478,
"needTime" : 0,
"needYield" : 0,
"saveState" : 2114,
"restoreState" : 2114,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"hubId" : 1,
"nodeId" : 1
},
"indexName" : "hubId_1_nodeId_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"hubId" : [
"[\"87-0c-4e-a6-ff-39-cc-94\", \"87-0c-4e-a6-ff-39-cc-94\"]"
],
"nodeId" : [
"[1.0, 1.0]"
]
},
"keysExamined" : 16478,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
},
{
"stage" : "IXSCAN",
"nReturned" : 0,
"executionTimeMillisEstimate" : 110,
"works" : 1,
"advanced" : 0,
"needTime" : 0,
"needYield" : 0,
"saveState" : 2114,
"restoreState" : 2114,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"neid" : 1,
"hubId" : 1,
"nodeId" : 1,
"capturedAt" : 1
},
"indexName" : "neid_1_hubId_1_nodeId_1_capturedAt_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"neid" : [
"[\"40-cb-28-dd-46-e7-d9-c3\", \"40-cb-28-dd-46-e7-d9-c3\"]"
],
"hubId" : [
"[MinKey, MaxKey]"
],
"nodeId" : [
"[MinKey, MaxKey]"
],
"capturedAt" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 0,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
]
}
}
}
},
"serverInfo" : {
"host" : "prod-clone",
"port" : 27017,
"version" : "3.2.4",
"gitVersion" : "e2ee9ffcf9f5a94fad76802e28cc978718bb7a30"
},
"ok" : 1
}
每个阶段的执行时间以毫秒为单位，总执行时间约为29秒（“executionTimeMillis”：28908）。这里的瓶颈是什么？如何优化索引？此服务器上有多少内存/cpu？每秒发送多少查询？mongodb是什么版本的？Mongo版本3.2.4。我用1vcpu和6.5gb内存在Google Compute实例上运行它。大量的内存是免费的。CPU成了瓶颈。现在还不清楚哪一个阶段是这里的瓶颈，我无法在不了解它的情况下进行优化。这个服务器上有多少内存/cpu？每秒发送多少查询？mongodb是什么版本的？Mongo版本3.2.4。我用1vcpu和6.5gb内存在Google Compute实例上运行它。大量的内存是免费的。CPU成了瓶颈。它非常不清楚哪个阶段是这里的瓶颈，我无法在不了解它的情况下进行优化。