Go 运行时：内存不足，内存静止

最近发生了一个非常奇怪的问题。我的程序运行在一个docker上，该docker有8GB内存，在发生恐慌时使用了4GB内存。有超过4GB的可用内存，为什么会发生这种情况？下面是紧急堆栈输出

# ulimit -m
unlimited

# go version
1.6.2

我试着反复运行代码行

getAllCombinationComplex

，使用的内存增加到5GB左右，但死机没有再次发生

fatal error: runtime: out of memory

runtime stack:
runtime.throw(0xd28a90, 0x16)
        /usr/local/go1.6.2/src/runtime/panic.go:547 +0x90
runtime.sysMap(0xf409e80000, 0x3ecc390000, 0x434a00, 0x1080918)
        /usr/local/go1.6.2/src/runtime/mem_linux.go:206 +0x9b
runtime.(*mheap).sysAlloc(0x1066280, 0x3ecc390000, 0xdfc4d6d680)
        /usr/local/go1.6.2/src/runtime/malloc.go:429 +0x191
runtime.(*mheap).grow(0x1066280, 0x1f661c8, 0x0)
        /usr/local/go1.6.2/src/runtime/mheap.go:651 +0x63
runtime.(*mheap).allocSpanLocked(0x1066280, 0x1f661c4, 0x10677f0)
        /usr/local/go1.6.2/src/runtime/mheap.go:553 +0x4f6
runtime.(*mheap).alloc_m(0x1066280, 0x1f661c4, 0xffffff0100000000, 0x7ff85dbfddd0)
        /usr/local/go1.6.2/src/runtime/mheap.go:437 +0x119
runtime.(*mheap).alloc.func1()
        /usr/local/go1.6.2/src/runtime/mheap.go:502 +0x41
runtime.systemstack(0x7ff85dbfdde8)
        /usr/local/go1.6.2/src/runtime/asm_amd64.s:307 +0xab
runtime.(*mheap).alloc(0x1066280, 0x1f661c4, 0x10100000000, 0x41587c)
        /usr/local/go1.6.2/src/runtime/mheap.go:503 +0x63
runtime.largeAlloc(0x3ecc386800, 0x7ff800000000, 0x6)
        /usr/local/go1.6.2/src/runtime/malloc.go:766 +0xb3
runtime.mallocgc.func3()
        /usr/local/go1.6.2/src/runtime/malloc.go:664 +0x33
runtime.systemstack(0xc820028000)
        /usr/local/go1.6.2/src/runtime/asm_amd64.s:291 +0x79
runtime.mstart()
        /usr/local/go1.6.2/src/runtime/proc.go:1051

goroutine 63798322149 [running]:
runtime.systemstack_switch()
        /usr/local/go1.6.2/src/runtime/asm_amd64.s:245 fp=0xea27f3c798 sp=0xea27f3c790
runtime.mallocgc(0x3ecc386800, 0xa572e0, 0x0, 0xec6c1e8000)
        /usr/local/go1.6.2/src/runtime/malloc.go:665 +0x9eb fp=0xea27f3c870 sp=0xea27f3c798
runtime.newarray(0xa572e0, 0x3ecc38680, 0x2)
        /usr/local/go1.6.2/src/runtime/malloc.go:798 +0xc9 fp=0xea27f3c8b0 sp=0xea27f3c870
runtime.makeslice(0xa38840, 0x3ecc38680, 0x3ecc38680, 0x0, 0x0, 0x0)
        /usr/local/go1.6.2/src/runtime/slice.go:32 +0x165 fp=0xea27f3c900 sp=0xea27f3c8b0
indexdb/db_template.XCludeList.getAllCombinationComplex(0xea4382c480, 0x8, 0x9, 0xe4ec1dc460, 0x4, 0x4, 0x0, 0x0, 0x0)
        /home/scmbuild/workspaces_cluster/index/src/indexdb/db_template/db.go:145 +0x371 fp=0xea27f3ca88
 sp=0xea27f3c900
indexdb/db_template.XCludeList.getAllCombinationComplex(0xea4382c480, 0x8, 0x9, 0xe4ec1dc438, 0x5, 0x5, 0x0, 0x0, 0x0)
        /home/scmbuild/workspaces_cluster/index/src/indexdb/db_template/db.go:141 +0x2ee fp=0xea27f3cc10
 sp=0xea27f3ca88
indexdb/db_template.XCludeList.getAllCombinationComplex(0xea4382c480, 0x8, 0x9, 0xe4ec1dc410, 0x6, 0x6, 0x0, 0x0, 0x0)
        /home/scmbuild/workspaces_cluster/index/src/indexdb/db_template/db.go:141 +0x2ee fp=0xea27f3cd98
 sp=0xea27f3cc10
indexdb/db_template.XCludeList.getAllCombinationComplex(0xea4382c480, 0x8, 0x9, 0xe4ec1dc3e8, 0x7, 0x7, 0x0, 0x0, 0x0)
        /home/scmbuild/workspaces_cluster/index/src/indexdb/db_template/db.go:141 +0x2ee fp=0xea27f3cf20
 sp=0xea27f3cd98
indexdb/db_template.XCludeList.getAllCombinationComplex(0xea4382c480, 0x8, 0x9, 0xe4ec1dc3c0, 0x8, 0x8, 0x0, 0x0, 0x0)
        /home/scmbuild/workspaces_cluster/index/src/indexdb/db_template/db.go:141 +0x2ee fp=0xea27f3d0a8
 sp=0xea27f3cf20
indexdb/db_template.XCludeList.GetAllCombinationString(0xea4382c480, 0x8, 0x9, 0x0, 0x0, 0x0)
        /home/scmbuild/workspaces_cluster/index/src/indexdb/db_template/db.go:114 +0x530 fp=0xea27f3d398
 sp=0xea27f3d0a8
indexdb/mem.MemDB.QueryCountersFullMatchByTags(0xead2bf0003, 0x38, 0xead2bf0043, 0x10, 0xea4382c480, 0x8, 0x9, 0x0, 0x0, 0x0, ...)
        /home/scmbuild/workspaces_cluster/index/src/indexdb/mem/mem.go:391 +0x1f5 fp=0xea27f3d670 sp=0xe
a27f3d398
indexdb/mem.(*MemDB).QueryCountersFullMatchByTags(0x107e7b8, 0xead2bf0003, 0x38, 0xead2bf0043, 0x10, 0xea4382c480, 0x8, 0x9, 0x0, 0x0, ...)
        <autogenerated>:10 +0x112 fp=0xea27f3d6d8 sp=0xea27f3d670
router.QCountersFullMatchByTags(0xd2745f5ef0)
        /home/scmbuild/workspaces_cluster/index/src/router/http.go:505 +0x54a fp=0xea27f3d8d8 sp=0xea27f
3d6d8
github.com/gin-gonic/gin.(*Context).Next(0xd2745f5ef0)
        /home/scmbuild/workspaces_cluster/index/deps/src/github.com/gin-gonic/gin/context.go:110 +0x7a f
p=0xea27f3d908 sp=0xea27f3d8d8
github.com/gin-gonic/gin.RecoveryWithWriter.func1(0xd2745f5ef0)
        /home/scmbuild/workspaces_cluster/index/deps/src/github.com/gin-gonic/gin/recovery.go:45 +0x51 f
p=0xea27f3d930 sp=0xea27f3d908
github.com/gin-gonic/gin.(*Context).Next(0xd2745f5ef0)
        /home/scmbuild/workspaces_cluster/index/deps/src/github.com/gin-gonic/gin/context.go:110 +0x7a f
p=0xea27f3d960 sp=0xea27f3d930
github.com/gin-gonic/gin.(*Engine).handleHTTPRequest(0xd4d0bd2360, 0xd2745f5ef0)
        /home/scmbuild/workspaces_cluster/index/deps/src/github.com/gin-gonic/gin/gin.go:337 +0x2fd fp=0
xea27f3dae8 sp=0xea27f3d960
github.com/gin-gonic/gin.(*Engine).ServeHTTP(0xd4d0bd2360, 0x7ff7611f00b0, 0xdb6777f1e0, 0xd6e25a0700)
        /home/scmbuild/workspaces_cluster/index/deps/src/github.com/gin-gonic/gin/gin.go:301 +0x197 fp=0
xea27f3db60 sp=0xea27f3dae8
net/http.serverHandler.ServeHTTP(0xd129858080, 0x7ff7611f00b0, 0xdb6777f1e0, 0xd6e25a0700)
        /usr/local/go1.6.2/src/net/http/server.go:2081 +0x19e fp=0xea27f3dbc0 sp=0xea27f3db60
net/http.(*conn).serve(0xce774f1400)
        /usr/local/go1.6.2/src/net/http/server.go:1472 +0xf2e fp=0xea27f3df88 sp=0xea27f3dbc0
runtime.goexit()
        /usr/local/go1.6.2/src/runtime/asm_amd64.s:1998 +0x1 fp=0xea27f3df90 sp=0xea27f3df88
created by net/http.(*Server).Serve
        /usr/local/go1.6.2/src/net/http/server.go:2137 +0x44e

致命错误：运行时：内存不足
运行时堆栈：
runtime.throw（0xd28a90，0x16）
/usr/local/go1.6.2/src/runtime/panic.go:547+0x90
sysMap（0xf409e80000、0x3ecc390000、0x434a00、0x1080918）
/usr/local/go1.6.2/src/runtime/mem_linux.go:206+0x9b
运行时。（*mheap）.sysAlloc（0x1066280，0x3ecc390000，0xDFC4D680）
/usr/local/go1.6.2/src/runtime/malloc.go:429+0x191
运行时。（*mheap）.grow（0x1066280，0x1f661c8，0x0）
/usr/local/go1.6.2/src/runtime/mheap.go:651+0x63
运行时。（*mheap）.allocSpanLocked（0x1066280、0x1f661c4、0x10677f0）
/usr/local/go1.6.2/src/runtime/mheap.go:553+0x4f6
运行时（*mheap）.allocm（0x1066280、0x1f661c4、0xFFFF0100000000、0x7ff85dbfddd0）
/usr/local/go1.6.2/src/runtime/mheap.go:437+0x119
运行时。（*mheap.alloc.func1（）
/usr/local/go1.6.2/src/runtime/mheap.go:502+0x41
runtime.systemstack（0x7ff85dbfdde8）
/usr/local/go1.6.2/src/runtime/asm_amd64.s:307+0xab
运行时（*mheap）.alloc（0x1066280、0x1f661c4、0x10100000000、0x41587c）
/usr/local/go1.6.2/src/runtime/mheap.go:503+0x63
runtime.largeAlloc（0x3ecc386800、0x7FF80000000、0x6）
/usr/local/go1.6.2/src/runtime/malloc.go:766+0xb3
runtime.mallocgc.func3（）
/usr/local/go1.6.2/src/runtime/malloc.go:664+0x33
runtime.systemstack（0xc820028000）
/usr/local/go1.6.2/src/runtime/asm_amd64.s:291+0x79
runtime.mstart（）
/usr/local/go1.6.2/src/runtime/proc.go:1051
goroutine 63798322149[运行]：
runtime.systemstack_开关（）
/usr/local/go1.6.2/src/runtime/asm_amd64.s:245 fp=0xea27f3c798 sp=0xea27f3c790
mallocgc（0x3ecc386800、0xa572e0、0x0、0xec6c1e8000）
/usr/local/go1.6.2/src/runtime/malloc.go:665+0x9eb fp=0xea27f3c870 sp=0xea27f3c798
runtime.newarray（0xa572e0、0x3ecc38680、0x2）
/usr/local/go1.6.2/src/runtime/malloc.go:798+0xc9 fp=0xea27f3c8b0 sp=0xea27f3c870
makeslice（0xa38840、0x3ecc38680、0x3ecc38680、0x0、0x0、0x0）
/usr/local/go1.6.2/src/runtime/slice.go:32+0x165 fp=0xea27f3c900 sp=0xea27f3c8b0
indexdb/db_template.XCludeList.getAllCombinationComplex（0xea4382c480、0x8、0x9、0xe4ec1dc460、0x4、0x4、0x0、0x0、0x0）
/home/scmbuild/workspaces_cluster/index/src/indexdb/db_template/db.go:145+0x371 fp=0xea27f3ca88
sp=0xea27f3c900
indexdb/db_template.XCludeList.getAllCombinationComplex（0xea4382c480、0x8、0x9、0xe4ec1dc438、0x5、0x5、0x0、0x0、0x0）
/home/scmbuild/workspaces_cluster/index/src/indexdb/db_template/db.go:141+0x2ee fp=0xea27f3cc10
sp=0xea27f3ca88
indexdb/db_template.XCludeList.getAllCombinationComplex（0xea4382c480、0x8、0x9、0xe4ec1dc410、0x6、0x6、0x0、0x0、0x0）
/home/scmbuild/workspaces_cluster/index/src/indexdb/db_template/db.go:141+0x2ee fp=0xea27f3cd98
sp=0xea27f3cc10
indexdb/db_template.XCludeList.getAllCombinationComplex（0xea4382c480、0x8、0x9、0xe4ec1dc3e8、0x7、0x0、0x0、0x0）
/home/scmbuild/workspaces_cluster/index/src/indexdb/db_template/db.go:141+0x2ee fp=0xea27f3cf20
sp=0xea27f3cd98
indexdb/db_template.XCludeList.getAllCombinationComplex（0xea4382c480、0x8、0x9、0xe4ec1dc3c0、0x8、0x8、0x0、0x0、0x0）
/home/scmbuild/workspaces_cluster/index/src/indexdb/db_template/db.go:141+0x2ee fp=0xea27f3d0a8
sp=0xea27f3cf20
indexdb/db_template.XCludeList.GetAllCombinationString（0xea4382c480、0x8、0x9、0x0、0x0、0x0）
/home/scmbuild/workspaces_cluster/index/src/indexdb/db_template/db.go:114+0x530 fp=0xea27f3d398
sp=0xea27f3d0a8
indexdb/mem.MemDB.QueryCountersFullMatchByTags（0xEAD2BF003、0x38、0xead2bf0043、0x10、0xea4382c480、0x8、0x9、0x0、0x0、0x0、…）
/home/scmbuild/workspaces\u cluster/index/src/indexdb/mem/mem.go:391+0x1f5 fp=0xea27f3d670 sp=0xe
a27f3d398
indexdb/mem（*MemDB）.QueryCountersFullMatchByTags（0x107e7b8，0xEAD2BF003，0x38，0xead2bf0043，0x10，0xea4382c480，0x8，0x9，0x0，0x0，…）
：10+0x112 fp=0xea27f3d6d8 sp=0xea27f3d670
router.qCounterFullMatchByTags（0xd2745f5ef0）
/home/scmbuild/workspaces\u cluster/index/src/router/http.go:505+0x54a fp=0xea27f3d8d8 sp=0xea27f
3d6d8
github.com/gin-gonic/gin.（*Context）.Next（0xd2745f5ef0）
/home/scmbuild/workspaces\u cluster/index/deps/src/github.com/gin-gonic/gin/context.go:110+0x7a f
p=0xea27f3d908 sp=0xea27f3d8d8
github.com/gin-gonic/gin.RecoveryWithWriter.func1（0xd2745f5ef0）
/home/scmbuild/workspaces\u cluster/index/deps/src/github.com/gin-gonic/gin/recovery.go:45+0x51 f
p=0xea27f3d930 sp=0xea27f3d908
github.com/gin-gonic/gin.（*Context）.Next（0xd2745f5ef0）
/home/scmbuild/workspaces\u cluster/index/deps/src/github.com/gin-gonic/gin/context.go:110+0x7a f
p=0xea27f3d960 sp=0xea27f3d930
github.com/gin-gonic/gin.（*Engine）.handleHTTPRequest（0xd4d0bd2360，0xd2745f5ef0）
/home/scmbuild/workspaces\u cluster/index/deps/src/github.com/gin-gonic/gin/gin.go:337+0x2fd fp=0
xea27f3dae8 sp=0xea27f3d960
github.com/gin-gonic/gin.（*Engine）.ServeHTTP（0xd4d0bd2360、0x7ff7611f00b0、0xdb6777f1e0、0xd6e25a0700）
/home/scmbuild/workspaces\u cluster/index/deps/src/github.com/gin-gonic/gin/gin.go:301+0x197 fp=0
xea27f3db60 sp=0xea27f3dae8
net/http.serverHandler.ServeHTTP（0xd129858080、0x7ff7611f00b0、0xdb6777f1e0、0xd6e25a0700）
/usr/local/go1.6.2/src/net/http/server.go:2081+0x19e fp=0xea27f3dbc0 sp=0xea27f3db60
net/http。（*conn）.serve（0xce774f1400）
/usr/local/go1.6.2/src/net/http/server.go:1472+0xf2e fp=0xea27f3df88 sp=0xea27f3dbc0
runtime.goexit（）
/usr/local/go1.6.2/src/runtime/asm_amd64.s:1998+0x1 fp=0xea27f3df90 sp=0xea27f3df88
由net/http创建。（*Server）.Serv
indexdb/db_template.XCludeList.getAllCombinationComplex(0xea4382c480, 0x8, 0x9, 0xe4ec1dc460, 0x4, 0x4, 0x0, 0x0, 0x0)
    /home/scmbuild/workspaces_cluster/index/src/indexdb/db_template/db.go:145 +0x371 fp=0xea27f3ca88 sp=0xea27f3c900

runtime.makeslice(0xa38840, 0x3ecc38680, 0x3ecc38680, 0x0, 0x0, 0x0)
    /usr/local/go1.6.2/src/runtime/slice.go:32 +0x165 fp=0xea27f3c900 sp=0xea27f3c8b0

func makeslice(t *slicetype, len64, cap64 int64) slice {
    ...
}

runtime.makeslice(0xa38840, 0x3ecc38680, 0x3ecc38680, 0x0, 0x0, 0x0)

0x3ecc38680 = 16857138816