Kafka list_clusters call returning no results despite receiving correct results

[jacklund]

Hi, here's my code:

use rusoto_core::Region;
use rusoto_kafka::{Kafka, KafkaClient, ListClustersRequest};
use std::error::Error;
use env_logger;

#[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
    let _ = env_logger::try_init();
    let client = KafkaClient::new(Region::UsEast1);
    let mut req = ListClustersRequest::default();
    req.cluster_name_filter = Some("kafka-log-".to_string());
    // req.max_results = Some(100);
    let res = client.list_clusters(req).await?;
    println!("{:?}", res);
    Ok(())
}

And my Cargo.toml:

[package]
name = "kafka-tail-logs"
version = "0.1.0"
edition = "2018"

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
clap = "2.33.3"
env_logger = "0.9.0"
log = "0.4.14"
rdkafka = "0.26.0"
rusoto_core = "0.47.0"
rusoto_kafka = "0.47.0"
tokio = { version = "1.11.0", features = ["full"] }

When I run this with debug logging, I get the following:

% RUST_LOG=debug ./target/debug/kafka-tail-logs
[2021-09-10T20:41:53Z DEBUG rusoto_core::request] Full request:
     method: GET
     final_uri: https://kafka.us-east-1.amazonaws.com/v1/clusters?clusterNameFilter=kafka-log-
    Headers:

[2021-09-10T20:41:53Z DEBUG rusoto_core::request] authorization:"AWS4-HMAC-SHA256 Credential=<redacted>/20210910/us-east-1/kafka/aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=<redacted>"
[2021-09-10T20:41:53Z DEBUG rusoto_core::request] content-length:"0"
[2021-09-10T20:41:53Z DEBUG rusoto_core::request] content-type:"application/x-amz-json-1.1"
[2021-09-10T20:41:53Z DEBUG rusoto_core::request] host:"kafka.us-east-1.amazonaws.com"
[2021-09-10T20:41:53Z DEBUG rusoto_core::request] x-amz-content-sha256:"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
[2021-09-10T20:41:53Z DEBUG rusoto_core::request] x-amz-date:"20210910T204153Z"
[2021-09-10T20:41:53Z DEBUG rusoto_core::request] x-amz-security-token:"<redacted>"
[2021-09-10T20:41:53Z DEBUG rusoto_core::request] user-agent:"rusoto/0.47.0 rust/1.55.0 linux"
[2021-09-10T20:41:53Z DEBUG rusoto_core::proto::json::payload] Response body: b"{\n  \"clusterInfoList\" : [ {\n    \"clusterName\" : \"kafka-log-qa\",\n    \"clusterArn\" : \"arn:aws:kafka:us-east-1:<redacted>:cluster/kafka-log-qa/<redacted>\",\n    \"currentVersion\" : \"KTVN6HI0JCMYU\",\n    \"creationTime\" : \"2021-02-25T22:03:17.503Z\",\n    \"state\" : \"ACTIVE\",\n    \"currentBrokerSoftwareInfo\" : {\n      \"kafkaVersion\" : \"2.5.1\",\n      \"configurationArn\" : \"arn:aws:kafka:us-east-1:<redacted>:configuration/kafka-log-qa-config/fec346ef-f3c6-466c-8fc9-105224a0789f-11\",\n      \"configurationRevision\" : 1\n    },\n    \"numberOfBrokerNodes\" : 6,\n    \"brokerNodeGroupInfo\" : {\n      \"instanceType\" : \"kafka.m5.large\",\n      \"brokerAZDistribution\" : \"DEFAULT\",\n      \"clientSubnets\" : [ \"subnet-32f19c44\", \"subnet-24189d7c\", \"subnet-6814a642\" ],\n      \"securityGroups\" : [ \"sg-0d9ad018432dadc98\" ],\n      \"storageInfo\" : {\n        \"ebsStorageInfo\" : {\n          \"volumeSize\" : 600\n        }\n      }\n    },\n    \"enhancedMonitoring\" : \"PER_TOPIC_PER_BROKER\",\n    \"openMonitoring\" : {\n      \"prometheus\" : {\n        \"jmxExporter\" : {\n          \"enabledInBroker\" : true\n        },\n        \"nodeExporter\" : {\n          \"enabledInBroker\" : true\n        }\n      }\n    },\n    \"zookeeperConnectString\" : \"z-3.kafka-log-qa.9a5bxx.c11.kafka.us-east-1.amazonaws.com:2181,z-1.kafka-log-qa.9a5bxx.c11.kafka.us-east-1.amazonaws.com:2181,z-2.kafka-log-qa.9a5bxx.c11.kafka.us-east-1.amazonaws.com:2181\",\n    \"zookeeperConnectStringTls\" : \"z-3.kafka-log-qa.9a5bxx.c11.kafka.us-east-1.amazonaws.com:2182,z-1.kafka-log-qa.9a5bxx.c11.kafka.us-east-1.amazonaws.com:2182,z-2.kafka-log-qa.9a5bxx.c11.kafka.us-east-1.amazonaws.com:2182\",\n    \"clientAuthentication\" : {\n      \"sasl\" : {\n        \"scram\" : {\n          \"enabled\" : true\n        }\n      }\n    },\n    \"encryptionInfo\" : {\n      \"encryptionAtRest\" : {\n        \"dataVolumeKMSKeyId\" : \"arn:aws:kms:us-east-1:<redacted>:key/7c508996-1723-4d7a-8c45-763d8553b221\"\n      },\n      \"encryptionInTransit\" : {\n        \"clientBroker\" : \"TLS\",\n        \"inCluster\" : true\n      }\n    },\n    \"loggingInfo\" : {\n      \"brokerLogs\" : {\n        \"s3\" : {\n          \"enabled\" : true,\n          \"bucket\" : \"kafka-log-qa-us-east-1-logs\"\n        },\n        \"cloudWatchLogs\" : {\n          \"enabled\" : false\n        },\n        \"firehose\" : {\n          \"enabled\" : false\n        }\n      }\n    },\n    \"tags\" : {\n      \"Role\" : \"kafka-log\",\n      \"Owner\" : \"team-streams\",\n      \"Dimension\" : \"qa\",\n      \"Application\" : \"kafka\",\n      \"Lifespan\" : \"permanent\"\n    }\n  } ]\n}"
[2021-09-10T20:41:53Z DEBUG rusoto_core::proto::json::payload] Response status: 200 OK
ListClustersResponse { cluster_info_list: None, next_token: None }

The response I get back from rusoto_kafka is empty, yet you can see from the debug output that it got the correct results.

One thing I can see from the response is that the JSON fields are different than expected, e.g. "clusterInfoList" instead of "ClusterInfoList". I'm not sure if this is an artifact of the debugging, because when I use the Amazon CLI, the field values are correct.