Crash during heavy subscribe/unsubscribe

[karalabe]

If a topic is subscribed to and immediately unsubscribed, there is probably a race where certain fields aren't initialized yet, leading to a null pointer error.

panic: runtime error: invalid memory address or nil pointer dereference
[signal 0xb code=0x1 addr=0x8 pc=0x5553ae]

goroutine 3169783 [running]:
runtime.panic(0x6166a0, 0x781273)
/opt/google/go/src/pkg/runtime/panic.c:279 +0xf5
github.com/project-iris/iris/container/queue.(_Queue).Reset(0x0)
/home/karalabe/work/iris/src/github.com/project-iris/iris/container/queue/queue.go:107 +0x1e
github.com/project-iris/iris/pool.(_ThreadPool).Terminate(0xc208056570, 0x63f101)
/home/karalabe/work/iris/src/github.com/project-iris/iris/pool/thread.go:84 +0x87
github.com/project-iris/iris/service/relay.(_relay).process(0xc20cf3c0c0)
/home/karalabe/work/iris/src/github.com/project-iris/iris/service/relay/proto.go:618 +0x16e
created by github.com/project-iris/iris/service/relay.(_Relay).acceptRelay
/home/karalabe/work/iris/src/github.com/project-iris/iris/service/relay/relay.go:126 +0x572

goroutine 16 [chan receive, 83 minutes]:
main.main()
/home/karalabe/work/iris/src/github.com/project-iris/iris/main.go:200 +0x914

goroutine 19 [finalizer wait]:
runtime.park(0x415a20, 0x7852c8, 0x783da9)
/opt/google/go/src/pkg/runtime/proc.c:1369 +0x89
runtime.parkunlock(0x7852c8, 0x783da9)
/opt/google/go/src/pkg/runtime/proc.c:1385 +0x3b
runfinq()
/opt/google/go/src/pkg/runtime/mgc0.c:2644 +0xcf
runtime.goexit()
/opt/google/go/src/pkg/runtime/proc.c:1445

goroutine 21 [chan receive]:
github.com/project-iris/iris/system.func·001()
/home/karalabe/work/iris/src/github.com/project-iris/iris/system/system.go:56 +0x54
created by github.com/project-iris/iris/system.init·1
/home/karalabe/work/iris/src/github.com/project-iris/iris/system/system.go:59 +0x3d

goroutine 22 [syscall, 83 minutes]:
os/signal.loop()
/opt/google/go/src/pkg/os/signal/signal_unix.go:21 +0x1e
created by os/signal.init·1
/opt/google/go/src/pkg/os/signal/signal_unix.go:27 +0x32

goroutine 23 [select]:
github.com/project-iris/iris/heart.(_Heart).beater(0xc208040910)
/home/karalabe/work/iris/src/github.com/project-iris/iris/heart/heart.go:131 +0x46b
created by github.com/project-iris/iris/heart.(_Heart).Start
/home/karalabe/work/iris/src/github.com/project-iris/iris/heart/heart.go:63 +0x2f

goroutine 24 [select, 83 minutes]:
github.com/project-iris/iris/proto/pastry.(_Overlay).acceptor(0xc20812e000, 0xc2080e8390, 0xc208044540)
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/pastry/handshake.go:81 +0xd4a
created by github.com/project-iris/iris/proto/pastry.(_Overlay).Boot
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/pastry/overlay.go:155 +0x339

goroutine 25 [select, 83 minutes]:
github.com/project-iris/iris/proto/pastry.(_Overlay).acceptor(0xc20812e000, 0xc2080e83c0, 0xc2080445a0)
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/pastry/handshake.go:81 +0xd4a
created by github.com/project-iris/iris/proto/pastry.(_Overlay).Boot
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/pastry/overlay.go:155 +0x339

goroutine 26 [select]:
github.com/project-iris/iris/proto/pastry.(_Overlay).manager(0xc20812e000)
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/pastry/maintenance.go:76 +0x1006
created by github.com/project-iris/iris/proto/pastry.(_Overlay).Boot
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/pastry/overlay.go:160 +0x3a3

goroutine 27 [select]:
github.com/project-iris/iris/heart.(_Heart).beater(0xc208040050)
/home/karalabe/work/iris/src/github.com/project-iris/iris/heart/heart.go:131 +0x46b
created by github.com/project-iris/iris/heart.(_Heart).Start
/home/karalabe/work/iris/src/github.com/project-iris/iris/heart/heart.go:63 +0x2f

goroutine 28 [IO wait]:
net.runtime_pollWait(0x7fc1f8e4a200, 0x72, 0x0)
/opt/google/go/src/pkg/runtime/netpoll.goc:146 +0x66
net.(_pollDesc).Wait(0xc2080e20d0, 0x72, 0x0, 0x0)
/opt/google/go/src/pkg/net/fd_poll_runtime.go:84 +0x46
net.(_pollDesc).WaitRead(0xc2080e20d0, 0x0, 0x0)
/opt/google/go/src/pkg/net/fd_poll_runtime.go:89 +0x42
net.(_netFD).accept(0xc2080e2070, 0x6b2e68, 0x0, 0x7fc1f8e433c8, 0xb)
/opt/google/go/src/pkg/net/fd_unix.go:419 +0x343
net.(_TCPListener).AcceptTCP(0xc208076028, 0xecbacce57, 0x0, 0x0)
/opt/google/go/src/pkg/net/tcpsock_posix.go:234 +0x5d
github.com/project-iris/iris/proto/stream.(_Listener).accepter(0xc2080ee3e0, 0x3b9aca00)
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/stream/stream.go:98 +0x241
created by github.com/project-iris/iris/proto/stream.(_Listener).Accept
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/stream/stream.go:74 +0x39

goroutine 29 [select, 83 minutes]:
github.com/project-iris/iris/proto/session.(_Listener).accepter(0xc2080446c0, 0x3b9aca00)
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/session/handshake.go:128 +0x42b
created by github.com/project-iris/iris/proto/session.(_Listener).Accept
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/session/handshake.go:109 +0x58

goroutine 32 [IO wait]:
net.runtime_pollWait(0x7fc1f8e4a150, 0x72, 0x0)
/opt/google/go/src/pkg/runtime/netpoll.goc:146 +0x66
net.(_pollDesc).Wait(0xc20809c220, 0x72, 0x0, 0x0)
/opt/google/go/src/pkg/net/fd_poll_runtime.go:84 +0x46
net.(_pollDesc).WaitRead(0xc20809c220, 0x0, 0x0)
/opt/google/go/src/pkg/net/fd_poll_runtime.go:89 +0x42
net.(_netFD).accept(0xc20809c1c0, 0x6b2e68, 0x0, 0x7fc1f8e433c8, 0xb)
/opt/google/go/src/pkg/net/fd_unix.go:419 +0x343
net.(_TCPListener).AcceptTCP(0xc2080d0010, 0xecbacce57, 0x0, 0x0)
/opt/google/go/src/pkg/net/tcpsock_posix.go:234 +0x5d
github.com/project-iris/iris/proto/stream.(_Listener).accepter(0xc20800e720, 0x3b9aca00)
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/stream/stream.go:98 +0x241
created by github.com/project-iris/iris/proto/stream.(_Listener).Accept
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/stream/stream.go:74 +0x39

goroutine 33 [select, 83 minutes]:
github.com/project-iris/iris/proto/session.(_Listener).accepter(0xc208004120, 0x3b9aca00)
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/session/handshake.go:128 +0x42b
created by github.com/project-iris/iris/proto/session.(_Listener).Accept
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/session/handshake.go:109 +0x58

goroutine 30 [IO wait]:
net.runtime_pollWait(0x7fc1f8e4a0a0, 0x72, 0x0)
/opt/google/go/src/pkg/runtime/netpoll.goc:146 +0x66
net.(_pollDesc).Wait(0xc2080e2140, 0x72, 0x0, 0x0)
/opt/google/go/src/pkg/net/fd_poll_runtime.go:84 +0x46
net.(_pollDesc).WaitRead(0xc2080e2140, 0x0, 0x0)
/opt/google/go/src/pkg/net/fd_poll_runtime.go:89 +0x42
net.(_netFD).readFrom(0xc2080e20e0, 0x7fc1f45898a4, 0x5dc, 0x5dc, 0x0, 0x0, 0x0, 0x7fc1f8e433c8, 0xb)
/opt/google/go/src/pkg/net/fd_unix.go:269 +0x3db
net.(_UDPConn).ReadFromUDP(0xc208076030, 0x7fc1f45898a4, 0x5dc, 0x5dc, 0x0, 0x0, 0x0, 0x0)
/opt/google/go/src/pkg/net/udpsock_posix.go:67 +0x129
github.com/project-iris/iris/proto/bootstrap.(_Bootstrapper).accept(0xc2080e0480)
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/bootstrap/bootstrap.go:198 +0x24a
created by github.com/project-iris/iris/proto/bootstrap.(_Bootstrapper).Boot
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/bootstrap/bootstrap.go:141 +0x60

goroutine 31 [select]:
github.com/project-iris/iris/proto/bootstrap.(_Bootstrapper).probe(0xc2080e0480)
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/bootstrap/bootstrap.go:267 +0x680
created by github.com/project-iris/iris/proto/bootstrap.(_Bootstrapper).Boot
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/bootstrap/bootstrap.go:142 +0x78

goroutine 48 [chan receive, 82 minutes]:
github.com/project-iris/iris/proto/bootstrap.(_Bootstrapper).scan(0xc2080e0480)
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/bootstrap/bootstrap.go:349 +0x218
created by github.com/project-iris/iris/proto/bootstrap.(_Bootstrapper).Boot
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/bootstrap/bootstrap.go:143 +0x90

goroutine 64 [IO wait]:
net.runtime_pollWait(0x7fc1f8e49ff0, 0x72, 0x0)
/opt/google/go/src/pkg/runtime/netpoll.goc:146 +0x66
net.(_pollDesc).Wait(0xc20809c290, 0x72, 0x0, 0x0)
/opt/google/go/src/pkg/net/fd_poll_runtime.go:84 +0x46
net.(_pollDesc).WaitRead(0xc20809c290, 0x0, 0x0)
/opt/google/go/src/pkg/net/fd_poll_runtime.go:89 +0x42
net.(_netFD).readFrom(0xc20809c230, 0x7fc1f45378a4, 0x5dc, 0x5dc, 0x0, 0x0, 0x0, 0x7fc1f8e433c8, 0xb)
/opt/google/go/src/pkg/net/fd_unix.go:269 +0x3db
net.(_UDPConn).ReadFromUDP(0xc2080d0018, 0x7fc1f45378a4, 0x5dc, 0x5dc, 0x0, 0x0, 0x0, 0x0)
/opt/google/go/src/pkg/net/udpsock_posix.go:67 +0x129
github.com/project-iris/iris/proto/bootstrap.(_Bootstrapper).accept(0xc208080680)
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/bootstrap/bootstrap.go:198 +0x24a
created by github.com/project-iris/iris/proto/bootstrap.(_Bootstrapper).Boot
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/bootstrap/bootstrap.go:141 +0x60

goroutine 65 [select]:
github.com/project-iris/iris/proto/bootstrap.(_Bootstrapper).probe(0xc208080680)
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/bootstrap/bootstrap.go:267 +0x680
created by github.com/project-iris/iris/proto/bootstrap.(_Bootstrapper).Boot
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/bootstrap/bootstrap.go:142 +0x78

goroutine 66 [select]:
github.com/project-iris/iris/proto/bootstrap.(_Bootstrapper).scan(0xc208080680)
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/bootstrap/bootstrap.go:341 +0x59b
created by github.com/project-iris/iris/proto/bootstrap.(_Bootstrapper).Boot
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/bootstrap/bootstrap.go:143 +0x90

goroutine 34 [select, 83 minutes]:
github.com/project-iris/iris/proto/iris.(_Overlay).tunneler(0xc2080e2460, 0xc2081d6580, 0x10, 0x10, 0xc208004540, 0xc2080044e0)
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/iris/tunnel.go:87 +0x9cb
created by github.com/project-iris/iris/proto/iris.(_Overlay).Boot
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/iris/overlay.go:93 +0x416

goroutine 35 [IO wait]:
net.runtime_pollWait(0x7fc1f8e49f40, 0x72, 0x0)
/opt/google/go/src/pkg/runtime/netpoll.goc:146 +0x66
net.(_pollDesc).Wait(0xc20809de90, 0x72, 0x0, 0x0)
/opt/google/go/src/pkg/net/fd_poll_runtime.go:84 +0x46
net.(_pollDesc).WaitRead(0xc20809de90, 0x0, 0x0)
/opt/google/go/src/pkg/net/fd_poll_runtime.go:89 +0x42
net.(_netFD).accept(0xc20809de30, 0x6b2e68, 0x0, 0x7fc1f8e433c8, 0xb)
/opt/google/go/src/pkg/net/fd_unix.go:419 +0x343
net.(_TCPListener).AcceptTCP(0xc2080d0048, 0xecbacce57, 0x0, 0x0)
/opt/google/go/src/pkg/net/tcpsock_posix.go:234 +0x5d
github.com/project-iris/iris/proto/stream.(_Listener).accepter(0xc2081d89e0, 0x3b9aca00)
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/stream/stream.go:98 +0x241
created by github.com/project-iris/iris/proto/stream.(_Listener).Accept
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/stream/stream.go:74 +0x39

goroutine 67 [select, 50 minutes]:
github.com/project-iris/iris/proto/iris.(_Overlay).tunneler(0xc2080e2460, 0xc2081d6590, 0x10, 0x10, 0xc2080fe660, 0xc2080fe600)
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/iris/tunnel.go:87 +0x9cb
created by github.com/project-iris/iris/proto/iris.(_Overlay).Boot
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/iris/overlay.go:93 +0x416

goroutine 68 [IO wait]:
net.runtime_pollWait(0x7fc1f8e49e90, 0x72, 0x0)
/opt/google/go/src/pkg/runtime/netpoll.goc:146 +0x66
net.(_pollDesc).Wait(0xc2081e03e0, 0x72, 0x0, 0x0)
/opt/google/go/src/pkg/net/fd_poll_runtime.go:84 +0x46
net.(_pollDesc).WaitRead(0xc2081e03e0, 0x0, 0x0)
/opt/google/go/src/pkg/net/fd_poll_runtime.go:89 +0x42
net.(_netFD).accept(0xc2081e0380, 0x6b2e68, 0x0, 0x7fc1f8e433c8, 0xb)
/opt/google/go/src/pkg/net/fd_unix.go:419 +0x343
net.(_TCPListener).AcceptTCP(0xc2080fa2a8, 0xecbacce57, 0x0, 0x0)
/opt/google/go/src/pkg/net/tcpsock_posix.go:234 +0x5d
github.com/project-iris/iris/proto/stream.(_Listener).accepter(0xc2081e2a00, 0x3b9aca00)
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/stream/stream.go:98 +0x241
created by github.com/project-iris/iris/proto/stream.(_Listener).Accept
/home/karalabe/work/iris/src/github.com/project-iris/iris/proto/stream/stream.go:74 +0x39

goroutine 65 [syscall, 83 minutes]:
runtime.goexit()
/opt/google/go/src/pkg/runtime/proc.c:1445

goroutine 69 [IO wait]:
net.runtime_pollWait(0x7fc1f8e49de0, 0x72, 0x0)
/opt/google/go/src/pkg/runtime/netpoll.goc:146 +0x66
net.(_pollDesc).Wait(0xc2081e0450, 0x72, 0x0, 0x0)
/opt/google/go/src/pkg/net/fd_poll_runtime.go:84 +0x46
net.(_pollDesc).WaitRead(0xc2081e0450, 0x0, 0x0)
/opt/google/go/src/pkg/net/fd_poll_runtime.go:89 +0x42
net.(_netFD).accept(0xc2081e03f0, 0x6b2e68, 0x0, 0x7fc1f8e433c8, 0xb)
/opt/google/go/src/pkg/net/fd_unix.go:419 +0x343
net.(_TCPListener).AcceptTCP(0xc2080fa2b8, 0xecbacce58, 0x0, 0x0)
/opt/google/go/src/pkg/net/tcpsock_posix.go:234 +0x5d
net.(_TCPListener).Accept(0xc2080fa2b8, 0x0, 0x0, 0x0, 0x0)
/opt/google/go/src/pkg/net/tcpsock_posix.go:244 +0x4b
github.com/project-iris/iris/service/relay.(_Relay).acceptor(0xc2081b5b90)
/home/karalabe/work/iris/src/github.com/project-iris/iris/service/relay/service.go:101 +0x62d
created by github.com/project-iris/iris/service/relay.(*Relay).Boot
/home/karalabe/work/iris/src/github.com/project-iris/iris/service/relay/service.go:72 +0x9b

goroutine 3168815 [runnable]:
github.com/project-iris/iris/service/relay.(_relay).process(0xc208140180)
/home/karalabe/work/iris/src/github.com/project-iris/iris/service/relay/proto.go:570
created by github.com/project-iris/iris/service/relay.(_Relay).acceptRelay
/home/karalabe/work/iris/src/github.com/project-iris/iris/service/relay/relay.go:126 +0x572
exit status 2