Adding goroutine trace. One connection (top one) is stuck in netpoll inside write. This connection is between our own services, so don't expect it to be slow. The issue can be fixed by adding a write timeout but what could be causing it?
goroutine profile: total 32685
1 @ 0x42fd33 0x42a7ee 0x429cb0 0x629fba 0x62a086 0x62d084 0x640964 0x5e6306 0x73b75b 0x73b0b2 0x744660 0x5f13ea 0x5fbc12 0x5f8dd0 0x5ffd9f 0x48dcea 0x5f729f 0x460ae1
# 0x429cb0 net.runtime_pollWait+0x60 /usr/local/go/src/runtime/netpoll.go:160
# 0x629fba net.(*pollDesc).Wait+0x3a /usr/local/go/src/net/fd_poll_runtime.go:73
# 0x62a086 net.(*pollDesc).WaitWrite+0x36 /usr/local/go/src/net/fd_poll_runtime.go:82
# 0x62d084 net.(*netFD).Write+0x414 /usr/local/go/src/net/fd_unix.go:336
# 0x640964 net.(*conn).Write+0xe4 /usr/local/go/src/net/net.go:184
# 0x5e6306 github.com/moonfrog/heimdall/network.(*Messenger).WriteMessage+0x86 /home/rts/gopath/src/github.com/moonfrog/heimdall/network/messenger.go:40
# 0x73b75b github.com/moonfrog/heimdall/service.(*Service).sendPacket+0x55b /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:91
# 0x73b0b2 github.com/moonfrog/heimdall/service.(*Service).Send+0x442 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:79
# 0x744660 github.com/moonfrog/heimdall/service/manager.(*Sticky).Send+0x5e0 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/manager/sticky.go:91
# 0x5f13ea github.com/moonfrog/heimdall/serviceController.(*Controller).Send+0x22a /home/rts/gopath/src/github.com/moonfrog/heimdall/serviceController/controller.go:169
# 0x5fbc12 github.com/moonfrog/heimdall/client.(*Client).sendToService+0x482 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:452
# 0x5f8dd0 github.com/moonfrog/heimdall/client.(*Client).process+0xfd0 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:256
# 0x5ffd9f github.com/moonfrog/heimdall/client.(*Client).stateMachine.func1+0x3f /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:136
# 0x48dcea github.com/moonfrog/heimdall/common.Safe+0x3a /home/rts/gopath/src/github.com/moonfrog/heimdall/common/utils.go:49
# 0x5f729f github.com/moonfrog/heimdall/client.(*Client).stateMachine+0x1df /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:137
1 @ 0x460ae1
1148 @ 0x42fd33 0x42fdf4 0x43f877 0x43f506 0x6299cc 0x62b7d9 0x62ccdc 0x640964 0x5e6306 0x73b75b 0x73cb72 0x74475e 0x5f1ea3 0x491dac 0x491c99 0x5fa486 0x5f7467 0x460ae1
# 0x43f506 net.runtime_Semacquire+0x26 /usr/local/go/src/runtime/sema.go:52
# 0x6299cc net.(*fdMutex).RWLock+0x16c /usr/local/go/src/net/fd_mutex.go:143
# 0x62b7d9 net.(*netFD).writeLock+0x39 /usr/local/go/src/net/fd_unix.go:194
# 0x62ccdc net.(*netFD).Write+0x6c /usr/local/go/src/net/fd_unix.go:319
# 0x640964 net.(*conn).Write+0xe4 /usr/local/go/src/net/net.go:184
# 0x5e6306 github.com/moonfrog/heimdall/network.(*Messenger).WriteMessage+0x86 /home/rts/gopath/src/github.com/moonfrog/heimdall/network/messenger.go:40
# 0x73b75b github.com/moonfrog/heimdall/service.(*Service).sendPacket+0x55b /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:91
# 0x73cb72 github.com/moonfrog/heimdall/service.(*Service).OnDisconnect+0xb2 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:179
# 0x74475e github.com/moonfrog/heimdall/service/manager.(*Sticky).Disconnect+0x5e /home/rts/gopath/src/github.com/moonfrog/heimdall/service/manager/sticky.go:97
# 0x5f1ea3 github.com/moonfrog/heimdall/serviceController.(*Controller).OnDisconnect+0x393 /home/rts/gopath/src/github.com/moonfrog/heimdall/serviceController/controller.go:210
# 0x491dac github.com/moonfrog/heimdall/router.(*Router).onDisconnect+0x8c /home/rts/gopath/src/github.com/moonfrog/heimdall/router/router.go:151
# 0x491c99 github.com/moonfrog/heimdall/router.(*Router).UnregisterClient+0x309 /home/rts/gopath/src/github.com/moonfrog/heimdall/router/router.go:146
# 0x5fa486 github.com/moonfrog/heimdall/client.(*Client).disconnect+0x206 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:329
# 0x5f7467 github.com/moonfrog/heimdall/client.(*Client).stateMachine+0x3a7 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:150
11385 @ 0x42fd33 0x43e9f7 0x43df52 0x5fc6df 0x49181f 0x5faae9 0x5f94bf 0x5f8cdb 0x5ffd9f 0x48dcea 0x5f729f 0x460ae1
# 0x5fc6df github.com/moonfrog/heimdall/client.(*Client).ForceDisconnect+0xdf /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:502
# 0x49181f github.com/moonfrog/heimdall/router.(*Router).RegisterClient+0x7af /home/rts/gopath/src/github.com/moonfrog/heimdall/router/router.go:114
# 0x5faae9 github.com/moonfrog/heimdall/client.(*Client).join+0x5e9 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:379
# 0x5f94bf github.com/moonfrog/heimdall/client.(*Client).handleRouterMsg+0x61f /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:265
# 0x5f8cdb github.com/moonfrog/heimdall/client.(*Client).process+0xedb /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:254
# 0x5ffd9f github.com/moonfrog/heimdall/client.(*Client).stateMachine.func1+0x3f /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:136
# 0x48dcea github.com/moonfrog/heimdall/common.Safe+0x3a /home/rts/gopath/src/github.com/moonfrog/heimdall/common/utils.go:49
# 0x5f729f github.com/moonfrog/heimdall/client.(*Client).stateMachine+0x1df /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:137
1 @ 0x42fd33 0x42fdf4 0x43f877 0x43f4c6 0x4c6d64 0x6f9081 0x704b71 0x460ae1
# 0x43f4c6 sync.runtime_Semacquire+0x26 /usr/local/go/src/runtime/sema.go:47
# 0x4c6d64 sync.(*WaitGroup).Wait+0xb4 /usr/local/go/src/sync/waitgroup.go:127
# 0x6f9081 github.com/samuel/go-zookeeper/zk.(*Conn).loop+0x8c1 /home/rts/gopath/src/github.com/samuel/go-zookeeper/zk/conn.go:436
# 0x704b71 github.com/samuel/go-zookeeper/zk.Connect.func1+0x21 /home/rts/gopath/src/github.com/samuel/go-zookeeper/zk/conn.go:218
11343 @ 0x42fd33 0x43e9f7 0x43df52 0x5f70ab 0x460ae1
# 0x5f70ab github.com/moonfrog/heimdall/client.(*Client).readPump+0x70b /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:111
4332 @ 0x42fd33 0x43e9f7 0x43df52 0x5f6d54 0x460ae1
# 0x5f6d54 github.com/moonfrog/heimdall/client.(*Client).readPump+0x3b4 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:104
1438 @ 0x42fd33 0x42fdf4 0x43f877 0x43f506 0x6299cc 0x62b7d9 0x62ccdc 0x640964 0x5e6306 0x73b75b 0x73ca92 0x7447ee 0x5f1fd4 0x491897 0x5faae9 0x5f94bf 0x5f8cdb 0x5ffd9f 0x48dcea 0x5f729f 0x460ae1
# 0x43f506 net.runtime_Semacquire+0x26 /usr/local/go/src/runtime/sema.go:52
# 0x6299cc net.(*fdMutex).RWLock+0x16c /usr/local/go/src/net/fd_mutex.go:143
# 0x62b7d9 net.(*netFD).writeLock+0x39 /usr/local/go/src/net/fd_unix.go:194
# 0x62ccdc net.(*netFD).Write+0x6c /usr/local/go/src/net/fd_unix.go:319
# 0x640964 net.(*conn).Write+0xe4 /usr/local/go/src/net/net.go:184
# 0x5e6306 github.com/moonfrog/heimdall/network.(*Messenger).WriteMessage+0x86 /home/rts/gopath/src/github.com/moonfrog/heimdall/network/messenger.go:40
# 0x73b75b github.com/moonfrog/heimdall/service.(*Service).sendPacket+0x55b /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:91
# 0x73ca92 github.com/moonfrog/heimdall/service.(*Service).OnReconnect+0xb2 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:171
# 0x7447ee github.com/moonfrog/heimdall/service/manager.(*Sticky).Reconnect+0x5e /home/rts/gopath/src/github.com/moonfrog/heimdall/service/manager/sticky.go:105
# 0x5f1fd4 github.com/moonfrog/heimdall/serviceController.(*Controller).OnReconnect+0xd4 /home/rts/gopath/src/github.com/moonfrog/heimdall/serviceController/controller.go:218
# 0x491897 github.com/moonfrog/heimdall/router.(*Router).RegisterClient+0x827 /home/rts/gopath/src/github.com/moonfrog/heimdall/router/router.go:116
# 0x5faae9 github.com/moonfrog/heimdall/client.(*Client).join+0x5e9 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:379
# 0x5f94bf github.com/moonfrog/heimdall/client.(*Client).handleRouterMsg+0x61f /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:265
# 0x5f8cdb github.com/moonfrog/heimdall/client.(*Client).process+0xedb /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:254
# 0x5ffd9f github.com/moonfrog/heimdall/client.(*Client).stateMachine.func1+0x3f /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:136
# 0x48dcea github.com/moonfrog/heimdall/common.Safe+0x3a /home/rts/gopath/src/github.com/moonfrog/heimdall/common/utils.go:49
# 0x5f729f github.com/moonfrog/heimdall/client.(*Client).stateMachine+0x1df /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:137
18 @ 0x42fd33 0x43e9f7 0x43df52 0x73fbab 0x460ae1
# 0x73fbab github.com/moonfrog/heimdall/service.(*Service).startHB.func1+0x23b /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:241
1 @ 0x42fd33 0x42fdf4 0x4061ef 0x405d0b 0x587454 0x460ae1
# 0x587454 github.com/moonfrog/badger/lib/stats/serverside/golang.(*Client).startScheduler+0x94 /home/rts/gopath/src/github.com/moonfrog/badger/lib/stats/serverside/golang/client.go:182
1 @ 0x42fd33 0x42a7ee 0x429cb0 0x629fba 0x62a026 0x62dcbc 0x64bbfd 0x54d121 0x54c2b9 0x54c106 0x54c808 0x5807ac 0x460ae1
# 0x429cb0 net.runtime_pollWait+0x60 /usr/local/go/src/runtime/netpoll.go:160
# 0x629fba net.(*pollDesc).Wait+0x3a /usr/local/go/src/net/fd_poll_runtime.go:73
# 0x62a026 net.(*pollDesc).WaitRead+0x36 /usr/local/go/src/net/fd_poll_runtime.go:78
# 0x62dcbc net.(*netFD).accept+0x27c /usr/local/go/src/net/fd_unix.go:426
# 0x64bbfd net.(*TCPListener).AcceptTCP+0x4d /usr/local/go/src/net/tcpsock_posix.go:254
# 0x54d121 net/http.tcpKeepAliveListener.Accept+0x41 /usr/local/go/src/net/http/server.go:2427
# 0x54c2b9 net/http.(*Server).Serve+0x129 /usr/local/go/src/net/http/server.go:2117
# 0x54c106 net/http.(*Server).ListenAndServe+0x136 /usr/local/go/src/net/http/server.go:2098
# 0x54c808 net/http.ListenAndServe+0x98 /usr/local/go/src/net/http/server.go:2195
# 0x5807ac github.com/moonfrog/heimdall/debugger.(*Debugger).Start+0x55c /home/rts/gopath/src/github.com/moonfrog/heimdall/debugger/debugger.go:89
2361 @ 0x42fd33 0x42fdf4 0x43f877 0x43f4c6 0x4c65b8 0x5f2066 0x5f9d26 0x5f8cdb 0x5ffd9f 0x48dcea 0x5f729f 0x460ae1
# 0x43f4c6 sync.runtime_Semacquire+0x26 /usr/local/go/src/runtime/sema.go:47
# 0x4c65b8 sync.(*RWMutex).RLock+0x58 /usr/local/go/src/sync/rwmutex.go:37
# 0x5f2066 github.com/moonfrog/heimdall/serviceController.(*Controller).OnConnect+0x46 /home/rts/gopath/src/github.com/moonfrog/heimdall/serviceController/controller.go:224
# 0x5f9d26 github.com/moonfrog/heimdall/client.(*Client).handleRouterMsg+0xe86 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:284
# 0x5f8cdb github.com/moonfrog/heimdall/client.(*Client).process+0xedb /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:254
# 0x5ffd9f github.com/moonfrog/heimdall/client.(*Client).stateMachine.func1+0x3f /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:136
# 0x48dcea github.com/moonfrog/heimdall/common.Safe+0x3a /home/rts/gopath/src/github.com/moonfrog/heimdall/common/utils.go:49
# 0x5f729f github.com/moonfrog/heimdall/client.(*Client).stateMachine+0x1df /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:137
387 @ 0x42fd33 0x42fdf4 0x43f877 0x43f506 0x6299cc 0x62b7d9 0x62ccdc 0x640964 0x5e6306 0x73b75b 0x73b0b2 0x744660 0x5f13ea 0x5fbc12 0x5f8dd0 0x5ffd9f 0x48dcea 0x5f729f 0x460ae1
# 0x43f506 net.runtime_Semacquire+0x26 /usr/local/go/src/runtime/sema.go:52
# 0x6299cc net.(*fdMutex).RWLock+0x16c /usr/local/go/src/net/fd_mutex.go:143
# 0x62b7d9 net.(*netFD).writeLock+0x39 /usr/local/go/src/net/fd_unix.go:194
# 0x62ccdc net.(*netFD).Write+0x6c /usr/local/go/src/net/fd_unix.go:319
# 0x640964 net.(*conn).Write+0xe4 /usr/local/go/src/net/net.go:184
# 0x5e6306 github.com/moonfrog/heimdall/network.(*Messenger).WriteMessage+0x86 /home/rts/gopath/src/github.com/moonfrog/heimdall/network/messenger.go:40
# 0x73b75b github.com/moonfrog/heimdall/service.(*Service).sendPacket+0x55b /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:91
# 0x73b0b2 github.com/moonfrog/heimdall/service.(*Service).Send+0x442 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:79
# 0x744660 github.com/moonfrog/heimdall/service/manager.(*Sticky).Send+0x5e0 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/manager/sticky.go:91
# 0x5f13ea github.com/moonfrog/heimdall/serviceController.(*Controller).Send+0x22a /home/rts/gopath/src/github.com/moonfrog/heimdall/serviceController/controller.go:169
# 0x5fbc12 github.com/moonfrog/heimdall/client.(*Client).sendToService+0x482 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:452
# 0x5f8dd0 github.com/moonfrog/heimdall/client.(*Client).process+0xfd0 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:256
# 0x5ffd9f github.com/moonfrog/heimdall/client.(*Client).stateMachine.func1+0x3f /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:136
# 0x48dcea github.com/moonfrog/heimdall/common.Safe+0x3a /home/rts/gopath/src/github.com/moonfrog/heimdall/common/utils.go:49
# 0x5f729f github.com/moonfrog/heimdall/client.(*Client).stateMachine+0x1df /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:137
1 @ 0x42fd33 0x42fdf4 0x43f877 0x43f4c6 0x4c67b1 0x5f3391 0x460ae1
# 0x43f4c6 sync.runtime_Semacquire+0x26 /usr/local/go/src/runtime/sema.go:47
# 0x4c67b1 sync.(*RWMutex).Lock+0xa1 /usr/local/go/src/sync/rwmutex.go:88
# 0x5f3391 github.com/moonfrog/heimdall/serviceController.(*Controller).monitor+0xc1 /home/rts/gopath/src/github.com/moonfrog/heimdall/serviceController/controller.go:286
1 @ 0x42fd33 0x42fdf4 0x43f877 0x43f506 0x6299cc 0x62b7d9 0x62ccdc 0x640964 0x5e6306 0x73b75b 0x73fb4c 0x460ae1
# 0x43f506 net.runtime_Semacquire+0x26 /usr/local/go/src/runtime/sema.go:52
# 0x6299cc net.(*fdMutex).RWLock+0x16c /usr/local/go/src/net/fd_mutex.go:143
# 0x62b7d9 net.(*netFD).writeLock+0x39 /usr/local/go/src/net/fd_unix.go:194
# 0x62ccdc net.(*netFD).Write+0x6c /usr/local/go/src/net/fd_unix.go:319
# 0x640964 net.(*conn).Write+0xe4 /usr/local/go/src/net/net.go:184
# 0x5e6306 github.com/moonfrog/heimdall/network.(*Messenger).WriteMessage+0x86 /home/rts/gopath/src/github.com/moonfrog/heimdall/network/messenger.go:40
# 0x73b75b github.com/moonfrog/heimdall/service.(*Service).sendPacket+0x55b /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:91
# 0x73fb4c github.com/moonfrog/heimdall/service.(*Service).startHB.func1+0x1dc /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:252
1 @ 0x42fd33 0x43e9f7 0x43df52 0x4558a8 0x460ae1
# 0x42fd33 runtime.gopark+0x163 /usr/local/go/src/runtime/proc.go:263
# 0x43e9f7 runtime.selectgoImpl+0xa67 /usr/local/go/src/runtime/select.go:392
# 0x43df52 runtime.selectgo+0x12 /usr/local/go/src/runtime/select.go:215
# 0x4558a8 runtime.ensureSigM.func1+0x358 /usr/local/go/src/runtime/signal1_unix.go:279
2 @ 0x42fd33 0x42a7ee 0x429cb0 0x629fba 0x62a026 0x62dcbc 0x64bbfd 0x5e6f16 0x460ae1
# 0x429cb0 net.runtime_pollWait+0x60 /usr/local/go/src/runtime/netpoll.go:160
# 0x629fba net.(*pollDesc).Wait+0x3a /usr/local/go/src/net/fd_poll_runtime.go:73
# 0x62a026 net.(*pollDesc).WaitRead+0x36 /usr/local/go/src/net/fd_poll_runtime.go:78
# 0x62dcbc net.(*netFD).accept+0x27c /usr/local/go/src/net/fd_unix.go:426
# 0x64bbfd net.(*TCPListener).AcceptTCP+0x4d /usr/local/go/src/net/tcpsock_posix.go:254
# 0x5e6f16 github.com/moonfrog/heimdall/network.(*TCPServer).Start.func1+0x46 /home/rts/gopath/src/github.com/moonfrog/heimdall/network/tcpServer.go:22
12 @ 0x42fd33 0x42fdf4 0x43f877 0x43f506 0x6299cc 0x62b7d9 0x62ccdc 0x640964 0x5e6306 0x73b75b 0x73b0b2 0x744660 0x5f13ea 0x73c308 0x73f5cf 0x48dcea 0x73f89c 0x460ae1
# 0x43f506 net.runtime_Semacquire+0x26 /usr/local/go/src/runtime/sema.go:52
# 0x6299cc net.(*fdMutex).RWLock+0x16c /usr/local/go/src/net/fd_mutex.go:143
# 0x62b7d9 net.(*netFD).writeLock+0x39 /usr/local/go/src/net/fd_unix.go:194
# 0x62ccdc net.(*netFD).Write+0x6c /usr/local/go/src/net/fd_unix.go:319
# 0x640964 net.(*conn).Write+0xe4 /usr/local/go/src/net/net.go:184
# 0x5e6306 github.com/moonfrog/heimdall/network.(*Messenger).WriteMessage+0x86 /home/rts/gopath/src/github.com/moonfrog/heimdall/network/messenger.go:40
# 0x73b75b github.com/moonfrog/heimdall/service.(*Service).sendPacket+0x55b /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:91
# 0x73b0b2 github.com/moonfrog/heimdall/service.(*Service).Send+0x442 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:79
# 0x744660 github.com/moonfrog/heimdall/service/manager.(*Sticky).Send+0x5e0 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/manager/sticky.go:91
# 0x5f13ea github.com/moonfrog/heimdall/serviceController.(*Controller).Send+0x22a /home/rts/gopath/src/github.com/moonfrog/heimdall/serviceController/controller.go:169
# 0x73c308 github.com/moonfrog/heimdall/service.(*Service).process+0x8a8 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:142
# 0x73f5cf github.com/moonfrog/heimdall/service.(*Service).readPump.func1.1+0x3f /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:111
# 0x48dcea github.com/moonfrog/heimdall/common.Safe+0x3a /home/rts/gopath/src/github.com/moonfrog/heimdall/common/utils.go:49
# 0x73f89c github.com/moonfrog/heimdall/service.(*Service).readPump.func1+0x2bc /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:112
102 @ 0x42fd33 0x42fdf4 0x43f877 0x43f506 0x6299cc 0x62b7d9 0x62ccdc 0x640964 0x5e6306 0x73b75b 0x73cb72 0x74475e 0x7457a6 0x5f23d9 0x5f9d26 0x5f8cdb 0x5ffd9f 0x48dcea 0x5f729f 0x460ae1
# 0x43f506 net.runtime_Semacquire+0x26 /usr/local/go/src/runtime/sema.go:52
# 0x6299cc net.(*fdMutex).RWLock+0x16c /usr/local/go/src/net/fd_mutex.go:143
# 0x62b7d9 net.(*netFD).writeLock+0x39 /usr/local/go/src/net/fd_unix.go:194
# 0x62ccdc net.(*netFD).Write+0x6c /usr/local/go/src/net/fd_unix.go:319
# 0x640964 net.(*conn).Write+0xe4 /usr/local/go/src/net/net.go:184
# 0x5e6306 github.com/moonfrog/heimdall/network.(*Messenger).WriteMessage+0x86 /home/rts/gopath/src/github.com/moonfrog/heimdall/network/messenger.go:40
# 0x73b75b github.com/moonfrog/heimdall/service.(*Service).sendPacket+0x55b /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:91
# 0x73cb72 github.com/moonfrog/heimdall/service.(*Service).OnDisconnect+0xb2 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:179
# 0x74475e github.com/moonfrog/heimdall/service/manager.(*Sticky).Disconnect+0x5e /home/rts/gopath/src/github.com/moonfrog/heimdall/service/manager/sticky.go:97
# 0x7457a6 github.com/moonfrog/heimdall/service/manager.(*Sticky).Connect+0x856 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/manager/sticky.go:147
# 0x5f23d9 github.com/moonfrog/heimdall/serviceController.(*Controller).OnConnect+0x3b9 /home/rts/gopath/src/github.com/moonfrog/heimdall/serviceController/controller.go:235
# 0x5f9d26 github.com/moonfrog/heimdall/client.(*Client).handleRouterMsg+0xe86 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:284
# 0x5f8cdb github.com/moonfrog/heimdall/client.(*Client).process+0xedb /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:254
# 0x5ffd9f github.com/moonfrog/heimdall/client.(*Client).stateMachine.func1+0x3f /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:136
# 0x48dcea github.com/moonfrog/heimdall/common.Safe+0x3a /home/rts/gopath/src/github.com/moonfrog/heimdall/common/utils.go:49
# 0x5f729f github.com/moonfrog/heimdall/client.(*Client).stateMachine+0x1df /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:137
13 @ 0x42fd33 0x42fdf4 0x43f877 0x43f506 0x6299cc 0x62b7d9 0x62ccdc 0x640964 0x5e6306 0x73b75b 0x73cb72 0x74475e 0x5f1ea3 0x491dac 0x491c99 0x5fa486 0x5fa25a 0x5f8cdb 0x5ffd9f 0x48dcea 0x5f729f 0x460ae1
# 0x43f506 net.runtime_Semacquire+0x26 /usr/local/go/src/runtime/sema.go:52
# 0x6299cc net.(*fdMutex).RWLock+0x16c /usr/local/go/src/net/fd_mutex.go:143
# 0x62b7d9 net.(*netFD).writeLock+0x39 /usr/local/go/src/net/fd_unix.go:194
# 0x62ccdc net.(*netFD).Write+0x6c /usr/local/go/src/net/fd_unix.go:319
# 0x640964 net.(*conn).Write+0xe4 /usr/local/go/src/net/net.go:184
# 0x5e6306 github.com/moonfrog/heimdall/network.(*Messenger).WriteMessage+0x86 /home/rts/gopath/src/github.com/moonfrog/heimdall/network/messenger.go:40
# 0x73b75b github.com/moonfrog/heimdall/service.(*Service).sendPacket+0x55b /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:91
# 0x73cb72 github.com/moonfrog/heimdall/service.(*Service).OnDisconnect+0xb2 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:179
# 0x74475e github.com/moonfrog/heimdall/service/manager.(*Sticky).Disconnect+0x5e /home/rts/gopath/src/github.com/moonfrog/heimdall/service/manager/sticky.go:97
# 0x5f1ea3 github.com/moonfrog/heimdall/serviceController.(*Controller).OnDisconnect+0x393 /home/rts/gopath/src/github.com/moonfrog/heimdall/serviceController/controller.go:210
# 0x491dac github.com/moonfrog/heimdall/router.(*Router).onDisconnect+0x8c /home/rts/gopath/src/github.com/moonfrog/heimdall/router/router.go:151
# 0x491c99 github.com/moonfrog/heimdall/router.(*Router).UnregisterClient+0x309 /home/rts/gopath/src/github.com/moonfrog/heimdall/router/router.go:146
# 0x5fa486 github.com/moonfrog/heimdall/client.(*Client).disconnect+0x206 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:329
# 0x5fa25a github.com/moonfrog/heimdall/client.(*Client).handleRouterMsg+0x13ba /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:274
# 0x5f8cdb github.com/moonfrog/heimdall/client.(*Client).process+0xedb /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:254
# 0x5ffd9f github.com/moonfrog/heimdall/client.(*Client).stateMachine.func1+0x3f /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:136
# 0x48dcea github.com/moonfrog/heimdall/common.Safe+0x3a /home/rts/gopath/src/github.com/moonfrog/heimdall/common/utils.go:49
# 0x5f729f github.com/moonfrog/heimdall/client.(*Client).stateMachine+0x1df /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:137
7 @ 0x42fd33 0x42a7ee 0x429cb0 0x629fba 0x62a026 0x62be7a 0x640644 0x611d09 0x612b7a 0x612eb9 0x5e60f4 0x73f614 0x460ae1
# 0x429cb0 net.runtime_pollWait+0x60 /usr/local/go/src/runtime/netpoll.go:160
# 0x629fba net.(*pollDesc).Wait+0x3a /usr/local/go/src/net/fd_poll_runtime.go:73
# 0x62a026 net.(*pollDesc).WaitRead+0x36 /usr/local/go/src/net/fd_poll_runtime.go:78
# 0x62be7a net.(*netFD).Read+0x23a /usr/local/go/src/net/fd_unix.go:250
# 0x640644 net.(*conn).Read+0xe4 /usr/local/go/src/net/net.go:172
# 0x611d09 bufio.(*Reader).fill+0x1e9 /usr/local/go/src/bufio/bufio.go:97
# 0x612b7a bufio.(*Reader).ReadSlice+0x21a /usr/local/go/src/bufio/bufio.go:328
# 0x612eb9 bufio.(*Reader).ReadBytes+0xa9 /usr/local/go/src/bufio/bufio.go:406
# 0x5e60f4 github.com/moonfrog/heimdall/network.(*Messenger).ReadMessage+0x54 /home/rts/gopath/src/github.com/moonfrog/heimdall/network/messenger.go:29
# 0x73f614 github.com/moonfrog/heimdall/service.(*Service).readPump.func1+0x34 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:102
1 @ 0x42fd33 0x42fdf4 0x43f877 0x43f4c6 0x4c5704 0x4c6740 0x5f2d6a 0x5f5fcc 0x5f5dc4 0x5890f3 0x460ae1
# 0x43f4c6 sync.runtime_Semacquire+0x26 /usr/local/go/src/runtime/sema.go:47
# 0x4c5704 sync.(*Mutex).Lock+0x1c4 /usr/local/go/src/sync/mutex.go:83
# 0x4c6740 sync.(*RWMutex).Lock+0x30 /usr/local/go/src/sync/rwmutex.go:83
# 0x5f2d6a github.com/moonfrog/heimdall/serviceController.(*Controller).refreshServices+0x4a /home/rts/gopath/src/github.com/moonfrog/heimdall/serviceController/controller.go:257
# 0x5f5fcc github.com/moonfrog/heimdall/serviceController.(*Controller).init.func1+0x1ec /home/rts/gopath/src/github.com/moonfrog/heimdall/serviceController/controller.go:333
# 0x5f5dc4 github.com/moonfrog/heimdall/serviceController.loadConfig.func2+0x84 /home/rts/gopath/src/github.com/moonfrog/heimdall/serviceController/configServices.go:84
# 0x5890f3 github.com/moonfrog/badger/zootils.(*ZooKeeper).watchFunc+0x53 /home/rts/gopath/src/github.com/moonfrog/badger/zootils/zootils.go:62
1 @ 0x42fd33 0x43e9f7 0x43df52 0x5fc579 0x491911 0x5faae9 0x5f94bf 0x5f8cdb 0x5ffd9f 0x48dcea 0x5f729f 0x460ae1
# 0x5fc579 github.com/moonfrog/heimdall/client.(*Client).RejectConnection+0x369 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:495
# 0x491911 github.com/moonfrog/heimdall/router.(*Router).RegisterClient+0x8a1 /home/rts/gopath/src/github.com/moonfrog/heimdall/router/router.go:109
# 0x5faae9 github.com/moonfrog/heimdall/client.(*Client).join+0x5e9 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:379
# 0x5f94bf github.com/moonfrog/heimdall/client.(*Client).handleRouterMsg+0x61f /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:265
# 0x5f8cdb github.com/moonfrog/heimdall/client.(*Client).process+0xedb /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:254
# 0x5ffd9f github.com/moonfrog/heimdall/client.(*Client).stateMachine.func1+0x3f /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:136
# 0x48dcea github.com/moonfrog/heimdall/common.Safe+0x3a /home/rts/gopath/src/github.com/moonfrog/heimdall/common/utils.go:49
# 0x5f729f github.com/moonfrog/heimdall/client.(*Client).stateMachine+0x1df /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:137
