Summary
I'm getting an error from the garbage collector when I try json-iterator under load. I'm unmarshalling a large JSON file (2.7MB) repeatedly during a test into a regular struct. When the test is run with a long duration (=load), like -test.benchtime 10s
, the test will fail while the minio server panics with the output below.
I'm not sure where the problem is but I'd be very grateful for some assistance.
Setup
$ go version
go version go1.7.6 linux/amd64
$ uname -a
Linux ... 4.10.0-24-generic #28-Ubuntu SMP Wed Jun 14 08:14:34 UTC 2017 x86_64 x86_64 x86_64 GNU/Linux
The server has the jsoniter integration. It's in a PR (https://github.com/minio/minio/pull/4692). The integration is simple: https://github.com/brendanashworth/minio/commit/035bd53d12c1aa21b92cebbc7faf32f01ed91621. The JSON is retrieved and unmarshalled from multiple disks.
Runtime panic
runtime: pointer 0xc420a04000 to unallocated spanidx=0x502 span.base()=0xc4209e6000 span.limit=0xc420a06000 span.state=2
runtime: found in object at *(0xc4221f0000+0x126b0)
object=0xc4221f0000 k=0x62110f8 s.base()=0xc4221f0000 s.limit=0xc42227ba40 s.sizeclass=0 s.elemsize=573440
*(object+0) = 0xc422366000
*(object+8) = 0x6
*(object+16) = 0xc422366006
*(object+24) = 0x7
*(object+32) = 0xc42182a000
*(object+40) = 0x80
*(object+48) = 0xc422366010
*(object+56) = 0x6
*(object+64) = 0xc422366016
*(object+72) = 0x7
*(object+80) = 0xc42182a080
*(object+88) = 0x80
*(object+96) = 0xc422366020
*(object+104) = 0x6
*(object+112) = 0xc422366026
*(object+120) = 0x7
*(object+128) = 0xc42182a200
*(object+136) = 0x80
*(object+144) = 0xc422366030
*(object+152) = 0x6
*(object+160) = 0xc422366036
*(object+168) = 0x7
*(object+176) = 0xc42182a280
*(object+184) = 0x80
*(object+192) = 0xc422366040
*(object+200) = 0x6
*(object+208) = 0xc422366046
*(object+216) = 0x7
*(object+224) = 0xc42182a300
*(object+232) = 0x80
*(object+240) = 0xc422366050
*(object+248) = 0x6
*(object+256) = 0xc422366056
*(object+264) = 0x7
*(object+272) = 0xc42182a380
*(object+280) = 0x80
*(object+288) = 0xc422366060
*(object+296) = 0x6
*(object+304) = 0xc422366066
*(object+312) = 0x7
*(object+320) = 0xc42182a400
*(object+328) = 0x80
*(object+336) = 0xc422366070
*(object+344) = 0x6
*(object+352) = 0xc422366076
*(object+360) = 0x7
*(object+368) = 0xc42182a480
*(object+376) = 0x80
*(object+384) = 0xc422366080
*(object+392) = 0x6
*(object+400) = 0xc422366086
*(object+408) = 0x7
*(object+416) = 0xc42182a500
*(object+424) = 0x80
*(object+432) = 0xc422366090
*(object+440) = 0x7
*(object+448) = 0xc422366097
*(object+456) = 0x7
*(object+464) = 0xc42182a580
*(object+472) = 0x80
*(object+480) = 0xc4223660a0
*(object+488) = 0x7
*(object+496) = 0xc4223660a7
*(object+504) = 0x7
*(object+512) = 0xc42182a600
*(object+520) = 0x80
*(object+528) = 0xc4223660b0
*(object+536) = 0x7
*(object+544) = 0xc4223660b7
*(object+552) = 0x7
*(object+560) = 0xc42182a680
*(object+568) = 0x80
*(object+576) = 0xc4223660c0
*(object+584) = 0x7
*(object+592) = 0xc4223660c7
*(object+600) = 0x7
*(object+608) = 0xc42182a700
*(object+616) = 0x80
*(object+624) = 0xc4223660d0
*(object+632) = 0x7
*(object+640) = 0xc4223660d7
*(object+648) = 0x7
*(object+656) = 0xc42182a780
*(object+664) = 0x80
*(object+672) = 0xc4223660e0
*(object+680) = 0x7
*(object+688) = 0xc4223660e7
*(object+696) = 0x7
*(object+704) = 0xc42182a800
*(object+712) = 0x80
*(object+720) = 0xc4223660f0
*(object+728) = 0x7
*(object+736) = 0xc4223660f7
*(object+744) = 0x7
*(object+752) = 0xc42182a880
*(object+760) = 0x80
*(object+768) = 0xc422366100
*(object+776) = 0x7
*(object+784) = 0xc422366107
*(object+792) = 0x7
*(object+800) = 0xc42182a900
*(object+808) = 0x80
*(object+816) = 0xc422366110
*(object+824) = 0x7
*(object+832) = 0xc422366117
*(object+840) = 0x7
*(object+848) = 0xc42182a980
*(object+856) = 0x80
*(object+864) = 0xc422366120
*(object+872) = 0x7
*(object+880) = 0xc422366127
*(object+888) = 0x7
*(object+896) = 0xc42182aa00
*(object+904) = 0x80
*(object+912) = 0xc422366130
*(object+920) = 0x7
*(object+928) = 0xc422366137
*(object+936) = 0x7
*(object+944) = 0xc42182aa80
*(object+952) = 0x80
*(object+960) = 0xc422366140
*(object+968) = 0x7
*(object+976) = 0xc422366147
*(object+984) = 0x7
*(object+992) = 0xc42182ab00
*(object+1000) = 0x80
*(object+1008) = 0xc422366150
*(object+1016) = 0x7
...
*(object+75320) = 0x9
*(object+75328) = 0xc4204312f9
*(object+75336) = 0x7
*(object+75344) = 0xc4209dff00
*(object+75352) = 0x80
*(object+75360) = 0xc420431300
*(object+75368) = 0x9
*(object+75376) = 0xc420431309
*(object+75384) = 0x7
*(object+75392) = 0xc4209dff80
*(object+75400) = 0x80
*(object+75408) = 0xc420431310
*(object+75416) = 0x9
*(object+75424) = 0xc420431319
*(object+75432) = 0x7
*(object+75440) = 0xc420a04000 <==
*(object+75448) = 0x80
*(object+75456) = 0xc420431320
*(object+75464) = 0x9
*(object+75472) = 0xc420431329
*(object+75480) = 0x7
*(object+75488) = 0xc420a04080
*(object+75496) = 0x80
*(object+75504) = 0xc420431330
*(object+75512) = 0x9
*(object+75520) = 0xc420431339
*(object+75528) = 0x7
*(object+75536) = 0xc420a04100
*(object+75544) = 0x80
*(object+75552) = 0xc420431340
*(object+75560) = 0x9
...
fatal error: found bad pointer in Go heap (incorrect use of unsafe or cgo?)
goroutine 29 [running]:
runtime.throw(0x105a5d5, 0x3e)
/root/go/src/runtime/panic.go:566 +0x95 fp=0xc420049e38 sp=0xc420049e18
runtime.heapBitsForObject(0xc420a04000, 0xc4221f0000, 0x126b0, 0xc41ffde767, 0xc400000002, 0x7f47268e8dd0, 0xc420020728, 0x131)
/root/go/src/runtime/mbitmap.go:429 +0x2ba fp=0xc420049e90 sp=0xc420049e38
runtime.scanobject(0xc4221f0000, 0xc420020728)
/root/go/src/runtime/mgcmark.go:1175 +0x170 fp=0xc420049f10 sp=0xc420049e90
runtime.gcDrain(0xc420020728, 0x6)
/root/go/src/runtime/mgcmark.go:999 +0x15c fp=0xc420049f48 sp=0xc420049f10
runtime.gcBgMarkWorker(0xc42001f500)
/root/go/src/runtime/mgc.go:1458 +0x4ad fp=0xc420049fb8 sp=0xc420049f48
runtime.goexit()
/root/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc420049fc0 sp=0xc420049fb8
created by runtime.gcBgMarkStartWorkers
/root/go/src/runtime/mgc.go:1349 +0x98
goroutine 1 [select, 12 minutes]:
github.com/minio/minio/cmd.handleSignals()
/root/gopath/src/github.com/minio/minio/cmd/signals.go:53 +0x607
github.com/minio/minio/cmd.serverMain(0xc42030e160)
/root/gopath/src/github.com/minio/minio/cmd/server-main.go:223 +0x807
github.com/minio/minio/vendor/github.com/minio/cli.HandleAction(0xe6e9c0, 0x10cd1d0, 0xc42030e160, 0xc4201a7900, 0x0)
/root/gopath/src/github.com/minio/minio/vendor/github.com/minio/cli/app.go:499 +0x7c
github.com/minio/minio/vendor/github.com/minio/cli.Command.Run(0x10251f0, 0x6, 0x0, 0x0, 0x0, 0x0, 0x0, 0x103f2fa, 0x1c, 0x0, ...)
/root/gopath/src/github.com/minio/minio/vendor/github.com/minio/cli/command.go:214 +0xb9a
github.com/minio/minio/vendor/github.com/minio/cli.(*App).Run(0xc42019fba0, 0xc42000a0c0, 0x6, 0x6, 0x0, 0x0)
/root/gopath/src/github.com/minio/minio/vendor/github.com/minio/cli/app.go:260 +0x816
github.com/minio/minio/cmd.Main(0xc42000a0c0, 0x6, 0x6)
/root/gopath/src/github.com/minio/minio/cmd/main.go:145 +0x8d
main.main()
/root/gopath/src/github.com/minio/minio/main.go:68 +0x7b
goroutine 17 [syscall, 12 minutes]:
os/signal.signal_recv(0xc42004bfa8)
/root/go/src/runtime/sigqueue.go:116 +0x157
os/signal.loop()
/root/go/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.1
/root/go/src/os/signal/signal_unix.go:28 +0x41
goroutine 66 [IO wait]:
net.runtime_pollWait(0x7f47268e7948, 0x72, 0x0)
/root/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc4201a8290, 0x72, 0xc42004de78, 0xc4200141b0)
/root/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc4201a8290, 0x15bf440, 0xc4200141b0)
/root/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).accept(0xc4201a8230, 0x0, 0x15bc300, 0xc42215b5e0)
/root/go/src/net/fd_unix.go:419 +0x238
net.(*TCPListener).accept(0xc42018a018, 0x433deb, 0xc42004df28, 0x914838)
/root/go/src/net/tcpsock_posix.go:132 +0x2e
net.(*TCPListener).AcceptTCP(0xc42018a018, 0xc4201ea0c0, 0xc42018a030, 0xc420576060)
/root/go/src/net/tcpsock.go:209 +0x49
github.com/minio/minio/pkg/http.(*httpListener).start.func3(0xc42018a018, 0xc420576060)
/root/gopath/src/github.com/minio/minio/pkg/http/listener.go:197 +0x41
created by github.com/minio/minio/pkg/http.(*httpListener).start
/root/gopath/src/github.com/minio/minio/pkg/http/listener.go:211 +0x1cd
goroutine 9 [select, 12 minutes, locked to thread]:
runtime.gopark(0x10cf070, 0x0, 0x10251e4, 0x6, 0x18, 0x2)
/root/go/src/runtime/proc.go:259 +0x13a
runtime.selectgoImpl(0xc420496730, 0x0, 0x18)
/root/go/src/runtime/select.go:423 +0x1235
runtime.selectgo(0xc420496730)
/root/go/src/runtime/select.go:238 +0x1c
runtime.ensureSigM.func1()
/root/go/src/runtime/signal1_unix.go:304 +0x2f3
runtime.goexit()
/root/go/src/runtime/asm_amd64.s:2086 +0x1
goroutine 8 [chan receive]:
github.com/minio/minio/pkg/http.(*httpListener).Accept(0xc4201a82a0, 0x10ce9e0, 0xc420aaa280, 0x15c6c40, 0xc420450390)
/root/gopath/src/github.com/minio/minio/pkg/http/listener.go:217 +0x6d
net/http.(*Server).Serve(0xc420315860, 0x15c3400, 0xc4201a82a0, 0x0, 0x0)
/root/go/src/net/http/server.go:2273 +0x1ce
github.com/minio/minio/pkg/http.(*Server).Start(0xc420315860, 0x0, 0x0)
/root/gopath/src/github.com/minio/minio/pkg/http/server.go:116 +0x1cc
github.com/minio/minio/cmd.serverMain.func1()
/root/gopath/src/github.com/minio/minio/cmd/server-main.go:199 +0x2d
created by github.com/minio/minio/cmd.serverMain
/root/gopath/src/github.com/minio/minio/cmd/server-main.go:200 +0x62c
goroutine 15 [select, 12 minutes]:
github.com/minio/minio/pkg/objcache.(*Cache).StartGC.func1(0xc42069e060)
/root/gopath/src/github.com/minio/minio/pkg/objcache/objcache.go:295 +0x133
created by github.com/minio/minio/pkg/objcache.(*Cache).StartGC
/root/gopath/src/github.com/minio/minio/pkg/objcache/objcache.go:304 +0x3f
goroutine 84 [select, 12 minutes]:
github.com/minio/minio/pkg/objcache.(*Cache).StartGC.func1(0xc42000a180)
/root/gopath/src/github.com/minio/minio/pkg/objcache/objcache.go:295 +0x133
created by github.com/minio/minio/pkg/objcache.(*Cache).StartGC
/root/gopath/src/github.com/minio/minio/pkg/objcache/objcache.go:304 +0x3f
goroutine 1243 [GC assist wait]:
github.com/minio/minio/vendor/github.com/json-iterator/go.(*Iterator).ReadString(0xc4204c2070, 0xc4204c2070, 0x552)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_iter_string.go:15 +0x1ce
github.com/minio/minio/vendor/github.com/json-iterator/go.(*stringCodec).Decode(0x187aeb8, 0xc4207dff38, 0xc4204c2070)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_native.go:15 +0x2b
github.com/minio/minio/vendor/github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc420458b40, 0xc4207dff30, 0xc4204c2070)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_struct_decoder.go:911 +0x5c
github.com/minio/minio/vendor/github.com/json-iterator/go.(*fourFieldsStructDecoder).Decode(0xc42045c0a0, 0xc4207dff30, 0xc4204c2070)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_struct_decoder.go:577 +0x2aa
github.com/minio/minio/vendor/github.com/json-iterator/go.(*sliceDecoder).doDecode.func1(0xc4204c2070, 0xe5ad2c)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_slice.go:101 +0xa1
github.com/minio/minio/vendor/github.com/json-iterator/go.(*Iterator).ReadArrayCB(0xc4204c2070, 0xc42267d830, 0xe5ad20)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_iter_array.go:39 +0xae
github.com/minio/minio/vendor/github.com/json-iterator/go.(*sliceDecoder).doDecode(0xc4204503c0, 0xc4200924c8, 0xc4204c2070)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_slice.go:104 +0xe7
github.com/minio/minio/vendor/github.com/json-iterator/go.(*sliceDecoder).Decode(0xc4204503c0, 0xc4200924c8, 0xc4204c2070)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_slice.go:82 +0x49
github.com/minio/minio/vendor/github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc420459440, 0xc420092410, 0xc4204c2070)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_struct_decoder.go:911 +0x5c
github.com/minio/minio/vendor/github.com/json-iterator/go.(*sevenFieldsStructDecoder).Decode(0xc420198600, 0xc420092410, 0xc4204c2070)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_struct_decoder.go:705 +0x3a7
github.com/minio/minio/vendor/github.com/json-iterator/go.(*Iterator).ReadVal(0xc4204c2070, 0xf09740, 0xc420092410)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect.go:216 +0xea
github.com/minio/minio/vendor/github.com/json-iterator/go.(*frozenConfig).Unmarshal(0xc4201a6360, 0xc422e00000, 0x299cbb, 0x299ebb, 0xf09740, 0xc420092410, 0x0, 0x0)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_config.go:283 +0x1ab
github.com/minio/minio/vendor/github.com/json-iterator/go.Unmarshal(0xc422e00000, 0x299cbb, 0x299ebb, 0xf09740, 0xc420092410, 0x1d, 0xc4217ac010)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_adapter.go:16 +0x67
github.com/minio/minio/cmd.xlMetaV1UnmarshalJSON(0xc422e00000, 0x299cbb, 0x299ebb, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/root/gopath/src/github.com/minio/minio/cmd/xl-v1-utils.go:127 +0xae
github.com/minio/minio/cmd.readXLMeta(0x15ccd80, 0xc420450810, 0xc4207cfc65, 0x9, 0xc4207cfc6f, 0x3, 0x0, 0x0, 0x0, 0x0, ...)
/root/gopath/src/github.com/minio/minio/cmd/xl-v1-utils.go:151 +0x1a7
github.com/minio/minio/cmd.readAllXLMetadata.func1(0xc420644ac0, 0xc4203ed880, 0x4, 0x4, 0xc4207cfc65, 0x9, 0xc4207cfc6f, 0x3, 0xc420468c80, 0x4, ...)
/root/gopath/src/github.com/minio/minio/cmd/xl-v1-utils.go:181 +0xb1
created by github.com/minio/minio/cmd.readAllXLMetadata
/root/gopath/src/github.com/minio/minio/cmd/xl-v1-utils.go:186 +0x2af
goroutine 1241 [GC assist wait]:
reflect.unsafe_NewArray(0xf67f40, 0x200, 0xf67f40)
/root/go/src/runtime/malloc.go:803 +0x35
reflect.MakeSlice(0x15d00c0, 0xe5ad20, 0x101, 0x200, 0xe5ad20, 0xc4203e7a80, 0x97)
/root/go/src/reflect/value.go:2043 +0xe5
github.com/minio/minio/vendor/github.com/json-iterator/go.growOne(0xc4203143f8, 0x15d00c0, 0xe5ad20, 0x15d00c0, 0xf67f40)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_slice.go:127 +0x8a
github.com/minio/minio/vendor/github.com/json-iterator/go.(*sliceDecoder).doDecode.func1(0xc420454000, 0xc42031432c)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_slice.go:100 +0x68
github.com/minio/minio/vendor/github.com/json-iterator/go.(*Iterator).ReadArrayCB(0xc420454000, 0xc420685830, 0xe5ad20)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_iter_array.go:39 +0xae
github.com/minio/minio/vendor/github.com/json-iterator/go.(*sliceDecoder).doDecode(0xc4204503c0, 0xc4203143f8, 0xc420454000)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_slice.go:104 +0xe7
github.com/minio/minio/vendor/github.com/json-iterator/go.(*sliceDecoder).Decode(0xc4204503c0, 0xc4203143f8, 0xc420454000)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_slice.go:82 +0x49
github.com/minio/minio/vendor/github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc420459440, 0xc420314340, 0xc420454000)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_struct_decoder.go:911 +0x5c
github.com/minio/minio/vendor/github.com/json-iterator/go.(*sevenFieldsStructDecoder).Decode(0xc420198600, 0xc420314340, 0xc420454000)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_struct_decoder.go:705 +0x3a7
github.com/minio/minio/vendor/github.com/json-iterator/go.(*Iterator).ReadVal(0xc420454000, 0xf09740, 0xc420314340)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect.go:216 +0xea
github.com/minio/minio/vendor/github.com/json-iterator/go.(*frozenConfig).Unmarshal(0xc4201a6360, 0xc42184c000, 0x299cbb, 0x299ebb, 0xf09740, 0xc420314340, 0x0, 0x0)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_config.go:283 +0x1ab
github.com/minio/minio/vendor/github.com/json-iterator/go.Unmarshal(0xc42184c000, 0x299cbb, 0x299ebb, 0xf09740, 0xc420314340, 0x1d, 0xc4217b0000)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_adapter.go:16 +0x67
github.com/minio/minio/cmd.xlMetaV1UnmarshalJSON(0xc42184c000, 0x299cbb, 0x299ebb, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/root/gopath/src/github.com/minio/minio/cmd/xl-v1-utils.go:127 +0xae
github.com/minio/minio/cmd.readXLMeta(0x15ccd80, 0xc420450780, 0xc4207cfc65, 0x9, 0xc4207cfc6f, 0x3, 0x0, 0x0, 0x0, 0x0, ...)
/root/gopath/src/github.com/minio/minio/cmd/xl-v1-utils.go:151 +0x1a7
github.com/minio/minio/cmd.readAllXLMetadata.func1(0xc420644ac0, 0xc4203ed880, 0x4, 0x4, 0xc4207cfc65, 0x9, 0xc4207cfc6f, 0x3, 0xc420468c80, 0x4, ...)
/root/gopath/src/github.com/minio/minio/cmd/xl-v1-utils.go:181 +0xb1
created by github.com/minio/minio/cmd.readAllXLMetadata
/root/gopath/src/github.com/minio/minio/cmd/xl-v1-utils.go:186 +0x2af
goroutine 1242 [GC assist wait]:
reflect.unsafe_NewArray(0xf67f40, 0x200, 0xf67f40)
/root/go/src/runtime/malloc.go:803 +0x35
reflect.MakeSlice(0x15d00c0, 0xe5ad20, 0x101, 0x200, 0xe5ad20, 0xc4203f5a20, 0x97)
/root/go/src/reflect/value.go:2043 +0xe5
github.com/minio/minio/vendor/github.com/json-iterator/go.growOne(0xc4201383f8, 0x15d00c0, 0xe5ad20, 0x15d00c0, 0xf67f40)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_slice.go:127 +0x8a
github.com/minio/minio/vendor/github.com/json-iterator/go.(*sliceDecoder).doDecode.func1(0xc4200720e0, 0xc42013832c)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_slice.go:100 +0x68
github.com/minio/minio/vendor/github.com/json-iterator/go.(*Iterator).ReadArrayCB(0xc4200720e0, 0xc420aa9830, 0xe5ad20)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_iter_array.go:39 +0xae
github.com/minio/minio/vendor/github.com/json-iterator/go.(*sliceDecoder).doDecode(0xc4204503c0, 0xc4201383f8, 0xc4200720e0)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_slice.go:104 +0xe7
github.com/minio/minio/vendor/github.com/json-iterator/go.(*sliceDecoder).Decode(0xc4204503c0, 0xc4201383f8, 0xc4200720e0)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_slice.go:82 +0x49
github.com/minio/minio/vendor/github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc420459440, 0xc420138340, 0xc4200720e0)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_struct_decoder.go:911 +0x5c
github.com/minio/minio/vendor/github.com/json-iterator/go.(*sevenFieldsStructDecoder).Decode(0xc420198600, 0xc420138340, 0xc4200720e0)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_struct_decoder.go:705 +0x3a7
github.com/minio/minio/vendor/github.com/json-iterator/go.(*Iterator).ReadVal(0xc4200720e0, 0xf09740, 0xc420138340)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect.go:216 +0xea
github.com/minio/minio/vendor/github.com/json-iterator/go.(*frozenConfig).Unmarshal(0xc4201a6360, 0xc42395e000, 0x299cbb, 0x299ebb, 0xf09740, 0xc420138340, 0x0, 0x0)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_config.go:283 +0x1ab
github.com/minio/minio/vendor/github.com/json-iterator/go.Unmarshal(0xc42395e000, 0x299cbb, 0x299ebb, 0xf09740, 0xc420138340, 0x1d, 0xc4217ae000)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_adapter.go:16 +0x67
github.com/minio/minio/cmd.xlMetaV1UnmarshalJSON(0xc42395e000, 0x299cbb, 0x299ebb, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/root/gopath/src/github.com/minio/minio/cmd/xl-v1-utils.go:127 +0xae
github.com/minio/minio/cmd.readXLMeta(0x15ccd80, 0xc4204507e0, 0xc4207cfc65, 0x9, 0xc4207cfc6f, 0x3, 0x0, 0x0, 0x0, 0x0, ...)
/root/gopath/src/github.com/minio/minio/cmd/xl-v1-utils.go:151 +0x1a7
github.com/minio/minio/cmd.readAllXLMetadata.func1(0xc420644ac0, 0xc4203ed880, 0x4, 0x4, 0xc4207cfc65, 0x9, 0xc4207cfc6f, 0x3, 0xc420468c80, 0x4, ...)
/root/gopath/src/github.com/minio/minio/cmd/xl-v1-utils.go:181 +0xb1
created by github.com/minio/minio/cmd.readAllXLMetadata
/root/gopath/src/github.com/minio/minio/cmd/xl-v1-utils.go:186 +0x2af
goroutine 1240 [semacquire]:
sync.runtime_Semacquire(0xc420644acc)
/root/go/src/runtime/sema.go:47 +0x30
sync.(*WaitGroup).Wait(0xc420644ac0)
/root/go/src/sync/waitgroup.go:131 +0x97
github.com/minio/minio/cmd.readAllXLMetadata(0xc420190680, 0x4, 0x4, 0xc4207cfc65, 0x9, 0xc4207cfc6f, 0x3, 0xc420290c60, 0x2, 0x2, ...)
/root/gopath/src/github.com/minio/minio/cmd/xl-v1-utils.go:190 +0x2ed
github.com/minio/minio/cmd.xlObjects.GetObject(0xc42045a638, 0xc420190680, 0x4, 0x4, 0x2, 0x2, 0x2, 0x3, 0xc42044e580, 0xc42000a180, ...)
/root/gopath/src/github.com/minio/minio/cmd/xl-v1-object.go:153 +0x144
github.com/minio/minio/cmd.(*xlObjects).GetObject(0xc4205760c0, 0xc4207cfc65, 0x9, 0xc4207cfc6f, 0x3, 0x0, 0xc80000000, 0x15bd700, 0xc420468140, 0x1852c60, ...)
<autogenerated>:1052 +0xfa
github.com/minio/minio/cmd.objectAPIHandlers.GetObjectHandler(0x10cd138, 0x15c33c0, 0xc42031a600, 0xc420130870)
/root/gopath/src/github.com/minio/minio/cmd/object-handlers.go:160 +0x606
github.com/minio/minio/cmd.(objectAPIHandlers).GetObjectHandler-fm(0x15c33c0, 0xc42031a600, 0xc420130870)
/root/gopath/src/github.com/minio/minio/cmd/api-router.go:56 +0x4e
net/http.HandlerFunc.ServeHTTP(0xc4204652b0, 0x15c33c0, 0xc42031a600, 0xc420130870)
/root/go/src/net/http/server.go:1726 +0x44
github.com/minio/minio/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc4201e03c0, 0x15c33c0, 0xc42031a600, 0xc420130870)
/root/gopath/src/github.com/minio/minio/vendor/github.com/gorilla/mux/mux.go:107 +0x10d
github.com/minio/minio/cmd.pathValidityHandler.ServeHTTP(0x15b8f00, 0xc4201e03c0, 0x15c33c0, 0xc42031a600, 0xc4201305a0)
/root/gopath/src/github.com/minio/minio/cmd/generic-handlers.go:470 +0x18b
github.com/minio/minio/cmd.(*pathValidityHandler).ServeHTTP(0xc42044a2b0, 0x15c33c0, 0xc42031a600, 0xc4201305a0)
<autogenerated>:512 +0x79
github.com/minio/minio/cmd.httpStatsHandler.ServeHTTP(0x15bd840, 0xc42044a2b0, 0x15c5bc0, 0xc420138270, 0xc4201305a0)
/root/gopath/src/github.com/minio/minio/cmd/generic-handlers.go:409 +0xcb
github.com/minio/minio/cmd.(*httpStatsHandler).ServeHTTP(0xc42044a2c0, 0x15c5bc0, 0xc420138270, 0xc4201305a0)
<autogenerated>:511 +0x79
github.com/minio/minio/cmd.requestSizeLimitHandler.ServeHTTP(0x15bd780, 0xc42044a2c0, 0x404000000, 0x15c5bc0, 0xc420138270, 0xc4201305a0)
/root/gopath/src/github.com/minio/minio/cmd/generic-handlers.go:65 +0x100
github.com/minio/minio/cmd.(*requestSizeLimitHandler).ServeHTTP(0xc420525380, 0x15c5bc0, 0xc420138270, 0xc4201305a0)
<autogenerated>:503 +0x82
github.com/minio/minio/cmd.crossDomainPolicy.ServeHTTP(0x15bd8c0, 0xc420525380, 0x15c5bc0, 0xc420138270, 0xc4201305a0)
/root/gopath/src/github.com/minio/minio/cmd/crossdomain-xml-handler.go:51 +0x65
github.com/minio/minio/cmd.(*crossDomainPolicy).ServeHTTP(0xc42044a2d0, 0x15c5bc0, 0xc420138270, 0xc4201305a0)
<autogenerated>:347 +0x79
github.com/minio/minio/cmd.redirectHandler.ServeHTTP(0x15bd6c0, 0xc42044a2d0, 0x15c5bc0, 0xc420138270, 0xc4201305a0)
/root/gopath/src/github.com/minio/minio/cmd/generic-handlers.go:131 +0xc0
github.com/minio/minio/cmd.(*redirectHandler).ServeHTTP(0xc42044a2e0, 0x15c5bc0, 0xc420138270, 0xc4201305a0)
<autogenerated>:504 +0x79
github.com/minio/minio/cmd.minioPrivateBucketHandler.ServeHTTP(0x15bd880, 0xc42044a2e0, 0x15c5bc0, 0xc420138270, 0xc4201305a0)
/root/gopath/src/github.com/minio/minio/cmd/generic-handlers.go:177 +0x90
github.com/minio/minio/cmd.(*minioPrivateBucketHandler).ServeHTTP(0xc42044a2f0, 0x15c5bc0, 0xc420138270, 0xc4201305a0)
<autogenerated>:506 +0x79
github.com/minio/minio/cmd.cacheControlHandler.ServeHTTP(0x15bd800, 0xc42044a2f0, 0x15c5bc0, 0xc420138270, 0xc4201305a0)
/root/gopath/src/github.com/minio/minio/cmd/generic-handlers.go:158 +0x60
github.com/minio/minio/cmd.(*cacheControlHandler).ServeHTTP(0xc42044a300, 0x15c5bc0, 0xc420138270, 0xc4201305a0)
<autogenerated>:505 +0x79
github.com/minio/minio/cmd.timeValidityHandler.ServeHTTP(0x15bd680, 0xc42044a300, 0x15c5bc0, 0xc420138270, 0xc4201305a0)
/root/gopath/src/github.com/minio/minio/cmd/generic-handlers.go:247 +0x1e9
github.com/minio/minio/cmd.(*timeValidityHandler).ServeHTTP(0xc42044a310, 0x15c5bc0, 0xc420138270, 0xc4201305a0)
<autogenerated>:507 +0x79
github.com/minio/minio/vendor/github.com/rs/cors.(*Cors).Handler.func1(0x15c5bc0, 0xc420138270, 0xc4201305a0)
/root/gopath/src/github.com/minio/minio/vendor/github.com/rs/cors/cors.go:190 +0xe9
net/http.HandlerFunc.ServeHTTP(0xc420525420, 0x15c5bc0, 0xc420138270, 0xc4201305a0)
/root/go/src/net/http/server.go:1726 +0x44
github.com/minio/minio/cmd.resourceHandler.ServeHTTP(0x15be900, 0xc420525420, 0x15c5bc0, 0xc420138270, 0xc4201305a0)
/root/gopath/src/github.com/minio/minio/cmd/generic-handlers.go:360 +0xa2
github.com/minio/minio/cmd.(*resourceHandler).ServeHTTP(0xc42044a390, 0x15c5bc0, 0xc420138270, 0xc4201305a0)
<autogenerated>:508 +0x79
github.com/minio/minio/cmd.authHandler.ServeHTTP(0x15bd900, 0xc42044a390, 0x15c5bc0, 0xc420138270, 0xc4201305a0)
/root/gopath/src/github.com/minio/minio/cmd/auth-handler.go:229 +0x157
github.com/minio/minio/cmd.(*authHandler).ServeHTTP(0xc42044a3b0, 0x15c5bc0, 0xc420138270, 0xc4201305a0)
<autogenerated>:288 +0x79
github.com/minio/minio/pkg/http.(*Server).Start.func1(0x15c5bc0, 0xc420138270, 0xc4201305a0)
/root/gopath/src/github.com/minio/minio/pkg/http/server.go:107 +0x108
net/http.HandlerFunc.ServeHTTP(0xc4201ea100, 0x15c5bc0, 0xc420138270, 0xc4201305a0)
/root/go/src/net/http/server.go:1726 +0x44
net/http.serverHandler.ServeHTTP(0xc420315860, 0x15c5bc0, 0xc420138270, 0xc4201305a0)
/root/go/src/net/http/server.go:2202 +0x7d
net/http.(*conn).serve(0xc420aaa280, 0x15c6b80, 0xc420469bc0)
/root/go/src/net/http/server.go:1579 +0x4b7
created by net/http.(*Server).Serve
/root/go/src/net/http/server.go:2293 +0x44d
goroutine 1244 [GC assist wait]:
reflect.unsafe_NewArray(0xf67f40, 0xf42, 0xf67f40)
/root/go/src/runtime/malloc.go:803 +0x35
reflect.MakeSlice(0x15d00c0, 0xe5ad20, 0xc36, 0xf42, 0xe5ad20, 0xc420447460, 0x97)
/root/go/src/reflect/value.go:2043 +0xe5
github.com/minio/minio/vendor/github.com/json-iterator/go.growOne(0xc4218324c8, 0x15d00c0, 0xe5ad20, 0x15d00c0, 0xf67f40)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_slice.go:127 +0x8a
github.com/minio/minio/vendor/github.com/json-iterator/go.(*sliceDecoder).doDecode.func1(0xc420072a80, 0xe5ad2c)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_slice.go:100 +0x68
github.com/minio/minio/vendor/github.com/json-iterator/go.(*Iterator).ReadArrayCB(0xc420072a80, 0xc422681830, 0xe5ad20)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_iter_array.go:39 +0xae
github.com/minio/minio/vendor/github.com/json-iterator/go.(*sliceDecoder).doDecode(0xc4204503c0, 0xc4218324c8, 0xc420072a80)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_slice.go:104 +0xe7
github.com/minio/minio/vendor/github.com/json-iterator/go.(*sliceDecoder).Decode(0xc4204503c0, 0xc4218324c8, 0xc420072a80)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_slice.go:82 +0x49
github.com/minio/minio/vendor/github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc420459440, 0xc421832410, 0xc420072a80)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_struct_decoder.go:911 +0x5c
github.com/minio/minio/vendor/github.com/json-iterator/go.(*sevenFieldsStructDecoder).Decode(0xc420198600, 0xc421832410, 0xc420072a80)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect_struct_decoder.go:705 +0x3a7
github.com/minio/minio/vendor/github.com/json-iterator/go.(*Iterator).ReadVal(0xc420072a80, 0xf09740, 0xc421832410)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_reflect.go:216 +0xea
github.com/minio/minio/vendor/github.com/json-iterator/go.(*frozenConfig).Unmarshal(0xc4201a6360, 0xc423bf8000, 0x299cbb, 0x299ebb, 0xf09740, 0xc421832410, 0x0, 0x0)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_config.go:283 +0x1ab
github.com/minio/minio/vendor/github.com/json-iterator/go.Unmarshal(0xc423bf8000, 0x299cbb, 0x299ebb, 0xf09740, 0xc421832410, 0x1d, 0xc420644ae0)
/root/gopath/src/github.com/minio/minio/vendor/github.com/json-iterator/go/feature_adapter.go:16 +0x67
github.com/minio/minio/cmd.xlMetaV1UnmarshalJSON(0xc423bf8000, 0x299cbb, 0x299ebb, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/root/gopath/src/github.com/minio/minio/cmd/xl-v1-utils.go:127 +0xae
github.com/minio/minio/cmd.readXLMeta(0x15ccd80, 0xc420450840, 0xc4207cfc65, 0x9, 0xc4207cfc6f, 0x3, 0x0, 0x0, 0x0, 0x0, ...)
/root/gopath/src/github.com/minio/minio/cmd/xl-v1-utils.go:151 +0x1a7
github.com/minio/minio/cmd.readAllXLMetadata.func1(0xc420644ac0, 0xc4203ed880, 0x4, 0x4, 0xc4207cfc65, 0x9, 0xc4207cfc6f, 0x3, 0xc420468c80, 0x4, ...)
/root/gopath/src/github.com/minio/minio/cmd/xl-v1-utils.go:181 +0xb1
created by github.com/minio/minio/cmd.readAllXLMetadata
/root/gopath/src/github.com/minio/minio/cmd/xl-v1-utils.go:186 +0x2af