Burrow failed to connect kafka cluster (kafka: client has run out of available brokers to talk to)

[swapnildangorechc]

Hi,

We have issue with burrow, which fails to connect kafka cluster, with below error.

Kafka.log

We see below info in server.log file.

[2022-07-26 15:01:24,823] INFO [SocketServer brokerId=1] Failed authentication with /xx.xx.xx.xx (SSL handshake failed) (org.apache.kafka.common.network.Selector)

Burrow.log

{"level":"info","ts":1657739261.2620974,"msg":"configuring","type":"coordinator","name":"httpserver"}
{"level":"info","ts":1657739261.2879257,"msg":"configuring","type":"coordinator","name":"notifier"}
{"level":"info","ts":1657739261.287961,"msg":"configuring","type":"coordinator","name":"cluster"}
{"level":"info","ts":1657739261.2879865,"msg":"configuring","type":"module","coordinator":"cluster","class":"kafka","name":"kafka"}
{"level":"info","ts":1657739261.2911863,"msg":"configuring","type":"coordinator","name":"consumer"}
{"level":"info","ts":1657739261.2912247,"msg":"configuring","type":"module","coordinator":"consumer","class":"kafka","name":"burrow"}
{"level":"info","ts":1657739261.2921846,"msg":"starting","type":"coordinator","name":"zookeeper"}
{"level":"info","ts":1657739261.3041592,"msg":"Connected to xx.xx.xx.xx:xxxx","type":"coordinator","name":"zookeeper"}
{"level":"info","ts":1657739261.321232,"msg":"Authenticated: id=144154910105734385, timeout=6000","type":"coordinator","name":"zookeeper"}
{"level":"info","ts":1657739261.3213053,"msg":"Re-submitting 0 credentials after reconnect","type":"coordinator","name":"zookeeper"}
{"level":"info","ts":1657739261.3518999,"msg":"starting","type":"coordinator","name":"storage"}
{"level":"info","ts":1657739261.351937,"msg":"starting","type":"module","coordinator":"storage","class":"inmemory","name":"default"}
{"level":"info","ts":1657739261.3519754,"msg":"starting","type":"coordinator","name":"evaluator"}
{"level":"info","ts":1657739261.3519819,"msg":"starting","type":"module","coordinator":"evaluator","class":"caching","name":"default"}
{"level":"info","ts":1657739261.3519928,"msg":"starting","type":"coordinator","name":"httpserver"}
{"level":"info","ts":1657739261.352134,"msg":"started listener","type":"coordinator","name":"httpserver","listener":"[::]:9100"}
{"level":"info","ts":1657739261.3521514,"msg":"starting","type":"coordinator","name":"notifier"}
{"level":"info","ts":1657739261.3521652,"msg":"starting","type":"coordinator","name":"cluster"}
{"level":"info","ts":1657739261.3521693,"msg":"starting","type":"module","coordinator":"cluster","class":"kafka","name":"kafka"}
{"level":"info","ts":1657739261.4988637,"msg":"starting evaluations","type":"coordinator","name":"notifier"}
{"level":"error","ts":1657739263.573983,"msg":"failed to start client","type":"module","coordinator":"cluster","class":"kafka","name":"kafka","error":"kafka: client has run out of available brokers to talk to (Is your cluster reachable?)"}
{"level":"info","ts":1657739263.5740325,"msg":"stopping","type":"coordinator","name":"notifier"}
{"level":"info","ts":1657739263.5740402,"msg":"shutdown","type":"coordinator","name":"httpserver"}
{"level":"info","ts":1657739263.5740795,"msg":"stopping","type":"coordinator","name":"evaluator"}
{"level":"info","ts":1657739263.5740893,"msg":"stopping","type":"module","coordinator":"evaluator","class":"caching","name":"default"}
{"level":"info","ts":1657739263.5740964,"msg":"stopping","type":"coordinator","name":"storage"}
{"level":"info","ts":1657739263.574107,"msg":"stopping","type":"module","coordinator":"storage","class":"inmemory","name":"default"}
{"level":"info","ts":1657739263.5741801,"msg":"stopping","type":"coordinator","name":"zookeeper"}
{"level":"info","ts":1657739263.5933099,"msg":"Recv loop terminated: err=EOF","type":"coordinator","name":"zookeeper"}
{"level":"info","ts":1657739263.5933375,"msg":"Send loop terminated: err=","type":"coordinator","name":"zookeeper"}

Burrow.toml

[general]
pidfile="burrow.pid"
stdout-logfile="burrow.out"

[logging]
filename="burrow.log"
level="info"
maxsize=100
maxbackups=5
maxage=7
use-localtime=true
use-compression=true

[zookeeper]
servers=[ "host:port","host:port","host:port" ]
timeout=6
root-path="/burrow/"

[client-profile.tlsonly]
client-id=""
kafka-version="0.10.2.1"
tls="tlsprofile"

[tls.tlsprofile]
certfile=""
keyfile=""
cafile=""
noverify=true

[cluster.kafka]
class-name="kafka"
servers=[ "host:port","host:port","host:port" ]
client-profile="tlsonly"
topic-refresh=120
offset-refresh=30

[consumer.burrow]
class-name="kafka"
cluster="kafka"
servers=[ "host:port","host:port","host:port" ]
client-profile="tlsonly"
group-blacklist="^(console-consumer-|python-kafka-consumer-|quick-).*$"
group-whitelist=""

[httpserver.burrowListener]
address=":9100"
tls="tlsprofile"

Any debugging help would be appriciated.

Regards,
Swapnil

[swapnildangorechc]

Kafka server.log

DEBUG [SslTransportLayer channelId=host:port:9093-host:port:49428-30 key=sun.nio.ch.SelectionKeyImpl@1cefb965] SSLEngine.closeInBound() raised an exception. (org.apache.kafka.common.network.SslTransportLayer)
javax.net.ssl.SSLException: Inbound closed before receiving peer's close_notify: possible truncation attack?
at sun.security.ssl.Alerts.getSSLException(Alerts.java:208)
at sun.security.ssl.SSLEngineImpl.fatal(SSLEngineImpl.java:1647)
at sun.security.ssl.SSLEngineImpl.fatal(SSLEngineImpl.java:1615)
at sun.security.ssl.SSLEngineImpl.closeInbound(SSLEngineImpl.java:1542)
at org.apache.kafka.common.network.SslTransportLayer.handshakeFailure(SslTransportLayer.java:871)
at org.apache.kafka.common.network.SslTransportLayer.maybeProcessHandshakeFailure(SslTransportLayer.java:909)
at org.apache.kafka.common.network.SslTransportLayer.handshake(SslTransportLayer.java:295)
at org.apache.kafka.common.network.KafkaChannel.prepare(KafkaChannel.java:173)
at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:543)
at org.apache.kafka.common.network.Selector.poll(Selector.java:481)
at kafka.network.Processor.poll(SocketServer.scala:923)
at kafka.network.Processor.run(SocketServer.scala:826)
at java.lang.Thread.run(Thread.java:748)

[swapnildangorechc]

Can anyone let us know how to debug this issue?

[f-sanges]

Hi @swapnildangorechc,
you should create certificates and put them here:
[tls.tlsprofile]
certfile=""
keyfile=""
cafile=""
noverify=true