Merge branch 'apache:master' into emit-pane

.github/actions/setup-default-test-properties/test-properties.json

@@ -14,7 +14,7 @@
   },
   "JavaTestProperties": {
     "SUPPORTED_VERSIONS": ["8", "11", "17"],
-    "FLINK_VERSIONS": ["1.13", "1.14", "1.15"],
+    "FLINK_VERSIONS": ["1.14", "1.15", "1.16", "1.17"],
     "SPARK_VERSIONS": ["2", "3"]
   },
   "GoTestProperties": {

.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml

@@ -18,21 +18,21 @@ name: PostCommit Java IO Performance Tests
 on:
   issue_comment:
     types: [created]
-#  schedule:
-#  - cron: '0 4/6 * * *'
+  schedule:
+  - cron: '0 4/6 * * *'
   pull_request_target:
     paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_IO_Performance_Tests.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
 permissions:
   actions: write
-  pull-requests: read
-  checks: read
+  pull-requests: write
+  checks: write
   contents: read
   deployments: read
   id-token: none
-  issues: read
+  issues: write
   discussions: read
   packages: read
   pages: read
@@ -127,4 +127,4 @@ jobs:
       with:
         commit: '${{ env.prsha || env.GITHUB_SHA }}'
         comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
-        files: '**/build/test-results/**/*.xml'
+        files: '**/build/test-results/**/*.xml'

.test-infra/kafka/proxy/README.md

@@ -0,0 +1,56 @@
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+-->
+
+# Kafka proxy
+
+Provisions a private IP bastion host on Google Cloud for use as a proxy to a private IP Kafka instance.
+
+# Prerequisites
+
+- Kafka cluster (See [.test-infra/kafka](..) for available solutions.)
+
+# Usage
+
+## Acquire bootstrap server hosts
+
+One of the variables requires a mapping of bootstrap server hosts to the desired proxy exposed port. See
+the variable description for `bootstrap_endpoint_mapping` found in the [variables.tf](variables.tf) file.
+
+## Apply module
+
+Follows typical terraform workflow without the use of a
+[backend](https://developer.hashicorp.com/terraform/language/settings/backends/configuration).
+
+```
+DIR=.test-infra/kafka/proxy
+terraform -chdir=$DIR init
+```
+
+```
+terraform -chdir=$DIR apply -var-file=common.tfvars -var-file=name_of_your_specific.tfvars
+```
+
+## Invoke gcloud ssh tunneling command
+
+Successful application of the module will output the specific gcloud command needed to tunnel the kafka traffic
+to your local machine. An example of such output would look similar to:
+
+```
+gcloud compute ssh yourinstance --tunnel-through-iap --project=project --zone=zone --ssh-flag="-4 -L9093:localhost:9093" --ssh-flag="-4 -L9092:localhost:9092" --ssh-flag="-4 -L9094:localhost:9094"
+```

.test-infra/kafka/proxy/common.tfvars

@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+allows_iap_ingress_cidr_range = "35.235.240.0/20"
+kafka_proxy_version           = "v0.3.8"
+machine_type                  = "n1-standard-1"

.test-infra/kafka/proxy/compute.tf

@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+
+  // builds kafka-proxy's bootstrap-server-mapping flag based on the provided bootstrap_endpoint_mapping variable.
+  bootstrap_server_mapping = join(" ", [
+    for k, v in var.bootstrap_endpoint_mapping :"--bootstrap-server-mapping=\"${k},0.0.0.0:${v}\""
+  ])
+
+  // builds the gcloud ssh-flag to output after provisioning the bastion host based on the provided bootstrap_endpoint_mapping variable.
+  ssh_flag = join(" ", [
+    for port in values(var.bootstrap_endpoint_mapping) :"--ssh-flag=\"-4 -L${port}:localhost:${port}\""
+  ])
+}
+
+// Provision a firewall allowing for Identity Aware Proxy (IAP) SSH traffic.
+resource "google_compute_firewall" "allow-ssh-ingress-from-iap" {
+  name      = "${data.google_compute_network.default.name}-allow-ssh-ingress-from-iap"
+  network   = data.google_compute_network.default.id
+  direction = "INGRESS"
+  allow {
+    protocol = "TCP"
+    ports    = [22]
+  }
+  source_ranges = [var.allows_iap_ingress_cidr_range]
+}
+
+// Generate a random string to use as a postfix for resource naming.
+resource "random_string" "postfix" {
+  length  = 4
+  upper   = false
+  lower   = true
+  numeric = true
+  special = false
+}
+
+// Provision a bastion host for use as a kafka proxy.
+// On launch, installs and executes https://github.com/grepplabs/kafka-proxy.
+resource "google_compute_instance" "bastion" {
+  machine_type = var.machine_type
+  name         = "kafka-proxy-${random_string.postfix.result}"
+  tags         = ["kafka-proxy"]
+  zone         = data.google_compute_zones.available.names[0]
+
+  boot_disk {
+    initialize_params {
+      image = "debian-cloud/debian-12"
+      size  = "10"
+    }
+  }
+  // Configures private IP only network interface.
+  network_interface {
+    subnetwork = data.google_compute_subnetwork.default.id
+  }
+  service_account {
+    email = data.google_service_account.default.email
+    // Defer to IAM roles
+    scopes = ["https://www.googleapis.com/auth/cloud-platform"]
+  }
+  metadata_startup_script = <<EOF
+$(gcloud info --format="value(basic.python_location)") -m pip install numpy
+curl -Ls https://github.com/grepplabs/kafka-proxy/releases/download/${var.kafka_proxy_version}/kafka-proxy-${var.kafka_proxy_version}-linux-amd64.tar.gz | tar xz
+./kafka-proxy server ${local.bootstrap_server_mapping}
+EOF
+}
+
+// Outputs the gcloud command to tunnel kafka proxy traffic to the local machine.
+output "gcloud_tunnel_command" {
+  value = <<EOF
+  gcloud compute ssh ${google_compute_instance.bastion.name} --tunnel-through-iap --project=${var.project} --zone=${google_compute_instance.bastion.zone} ${local.ssh_flag}
+EOF
+}

.test-infra/kafka/proxy/prerequisites.tf

@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+resource "google_project_service" "required" {
+  for_each = toset([
+    "compute",
+    "iam",
+  ])
+  service            = "${each.key}.googleapis.com"
+  disable_on_destroy = false
+}
+
+// Query the VPC network to make sure it exists.
+data "google_compute_network" "default" {
+  depends_on = [google_project_service.required]
+  name       = var.network
+}
+
+// Query the VPC subnetwork to make sure it exists in the region specified.
+data "google_compute_subnetwork" "default" {
+  depends_on = [google_project_service.required]
+  name       = var.subnetwork
+  region     = var.region
+  lifecycle {
+    postcondition {
+      condition     = self.private_ip_google_access
+      error_message = <<EOT
+fatal: ${self.id} misconfigured: private Google access disabled.
+See https://cloud.google.com/vpc/docs/configure-private-google-access for details.
+EOT
+    }
+  }
+}
+
+// Query the Google Compute Router to make sure it exists; needed to access resources outside the VPC network
+// for private nodes.
+data "google_compute_router" "default" {
+  name    = var.router
+  network = data.google_compute_network.default.name
+  region  = data.google_compute_subnetwork.default.region
+}
+
+// Query the Google Compute Router NAT to make sure it exists.
+data "google_compute_router_nat" "default" {
+  name   = var.router_nat
+  router = data.google_compute_router.default.name
+  region = data.google_compute_router.default.region
+}
+
+// Query the Service Account.
+data "google_service_account" "default" {
+  depends_on = [google_project_service.required]
+  account_id = var.service_account_id
+}
+
+// Query available zones in the Compute region.
+data "google_compute_zones" "available" {
+  region = data.google_compute_subnetwork.default.region
+}
+

.test-infra/kafka/proxy/provider.tf

@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Setup Google Cloud provider
+provider "google" {
+  project = var.project
+}