From b16bcdd0f80ba13842df21ac704e570f8bfa3b06 Mon Sep 17 00:00:00 2001
From: Sergii Tkachenko <sergiitk@google.com>
Date: Tue, 6 Sep 2022 16:06:49 -0700
Subject: [PATCH] xDS interop: enable pod log collection in the buildscripts
 (#9495) (#9513)

- Enables pod log collection in all PSM interop jobs implemented
  in https://github.com/grpc/grpc/pull/30594.
- Associate test suite runs with their own log file, so it's displayed
  on the "Target Log" tab
- Updates security job to not stop after a failed suite, so that
  authz_test run even if security_test failed
- Fix run_test not returning correct exit status, causing false
  positives in some cases. See https://github.com/grpc/grpc/pull/30768
---
 buildscripts/kokoro/psm-security.cfg |  2 +-
 buildscripts/kokoro/psm-security.sh  | 22 ++++++++++++++++------
 buildscripts/kokoro/xds_k8s_lb.cfg   |  2 +-
 buildscripts/kokoro/xds_k8s_lb.sh    | 10 +++++++---
 buildscripts/kokoro/xds_url_map.cfg  |  2 +-
 buildscripts/kokoro/xds_url_map.sh   | 10 +++++++---
 6 files changed, 33 insertions(+), 15 deletions(-)

diff --git a/buildscripts/kokoro/psm-security.cfg b/buildscripts/kokoro/psm-security.cfg
index f2cfd7babff..9df0bbb7867 100644
--- a/buildscripts/kokoro/psm-security.cfg
+++ b/buildscripts/kokoro/psm-security.cfg
@@ -7,7 +7,7 @@ timeout_mins: 180
 action {
   define_artifacts {
     regex: "artifacts/**/*sponge_log.xml"
-    regex: "artifacts/**/*sponge_log.log"
+    regex: "artifacts/**/*.log"
     strip_prefix: "artifacts"
   }
 }
diff --git a/buildscripts/kokoro/psm-security.sh b/buildscripts/kokoro/psm-security.sh
index 9e49c26ad6c..636daa81822 100755
--- a/buildscripts/kokoro/psm-security.sh
+++ b/buildscripts/kokoro/psm-security.sh
@@ -118,6 +118,8 @@ run_test() {
   # Test driver usage:
   # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage
   local test_name="${1:?Usage: run_test test_name}"
+  local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}"
+  mkdir -pv "${out_dir}"
   set -x
   python -m "tests.${test_name}" \
     --flagfile="${TEST_DRIVER_FLAGFILE}" \
@@ -125,9 +127,11 @@ run_test() {
     --server_image="${SERVER_IMAGE_NAME}:${GIT_COMMIT}" \
     --client_image="${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" \
     --testing_version="${TESTING_VERSION}" \
-    --xml_output_file="${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml" \
-    --force_cleanup
-  set +x
+    --force_cleanup \
+    --collect_app_logs \
+    --log_dir="${out_dir}" \
+    --xml_output_file="${out_dir}/sponge_log.xml" \
+    |& tee "${out_dir}/sponge_log.log"
 }
 
 #######################################
@@ -169,9 +173,15 @@ main() {
   build_docker_images_if_needed
   # Run tests
   cd "${TEST_DRIVER_FULL_DIR}"
-  run_test baseline_test
-  run_test security_test
-  run_test authz_test
+  local failed_tests=0
+  test_suites=("baseline_test" "security_test" "authz_test")
+  for test in "${test_suites[@]}"; do
+    run_test $test || (( failed_tests++ ))
+  done
+  echo "Failed test suites: ${failed_tests}"
+  if (( failed_tests > 0 )); then
+    exit 1
+  fi
 }
 
 main "$@"
diff --git a/buildscripts/kokoro/xds_k8s_lb.cfg b/buildscripts/kokoro/xds_k8s_lb.cfg
index 43971896cd4..10ea2d43b5d 100644
--- a/buildscripts/kokoro/xds_k8s_lb.cfg
+++ b/buildscripts/kokoro/xds_k8s_lb.cfg
@@ -7,7 +7,7 @@ timeout_mins: 180
 action {
   define_artifacts {
     regex: "artifacts/**/*sponge_log.xml"
-    regex: "artifacts/**/*sponge_log.log"
+    regex: "artifacts/**/*.log"
     strip_prefix: "artifacts"
   }
 }
diff --git a/buildscripts/kokoro/xds_k8s_lb.sh b/buildscripts/kokoro/xds_k8s_lb.sh
index f36394a0c49..42ab9e8e8fd 100755
--- a/buildscripts/kokoro/xds_k8s_lb.sh
+++ b/buildscripts/kokoro/xds_k8s_lb.sh
@@ -117,6 +117,8 @@ run_test() {
   # Test driver usage:
   # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage
   local test_name="${1:?Usage: run_test test_name}"
+  local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}"
+  mkdir -pv "${out_dir}"
   set -x
   python -m "tests.${test_name}" \
     --flagfile="${TEST_DRIVER_FLAGFILE}" \
@@ -125,9 +127,11 @@ run_test() {
     --server_image="${SERVER_IMAGE_NAME}:${GIT_COMMIT}" \
     --client_image="${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" \
     --testing_version="${TESTING_VERSION}" \
-    --xml_output_file="${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml" \
-    --force_cleanup
-  set +x
+    --force_cleanup \
+    --collect_app_logs \
+    --log_dir="${out_dir}" \
+    --xml_output_file="${out_dir}/sponge_log.xml" \
+    |& tee "${out_dir}/sponge_log.log"
 }
 
 #######################################
diff --git a/buildscripts/kokoro/xds_url_map.cfg b/buildscripts/kokoro/xds_url_map.cfg
index 4b5be84f880..1fa6c0141cb 100644
--- a/buildscripts/kokoro/xds_url_map.cfg
+++ b/buildscripts/kokoro/xds_url_map.cfg
@@ -7,7 +7,7 @@ timeout_mins: 90
 action {
   define_artifacts {
     regex: "artifacts/**/*sponge_log.xml"
-    regex: "artifacts/**/*sponge_log.log"
+    regex: "artifacts/**/*.log"
     strip_prefix: "artifacts"
   }
 }
diff --git a/buildscripts/kokoro/xds_url_map.sh b/buildscripts/kokoro/xds_url_map.sh
index b2d2e373d02..3ef2d539308 100755
--- a/buildscripts/kokoro/xds_url_map.sh
+++ b/buildscripts/kokoro/xds_url_map.sh
@@ -117,15 +117,19 @@ run_test() {
   # Test driver usage:
   # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage
   local test_name="${1:?Usage: run_test test_name}"
+  local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}"
+  mkdir -pv "${out_dir}"
   set -x
   python -m "tests.${test_name}" \
     --flagfile="${TEST_DRIVER_FLAGFILE}" \
+    --flagfile="config/url-map.cfg" \
     --kube_context="${KUBE_CONTEXT}" \
     --client_image="${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" \
     --testing_version="${TESTING_VERSION}" \
-    --xml_output_file="${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml" \
-    --flagfile="config/url-map.cfg"
-  set +x
+    --collect_app_logs \
+    --log_dir="${out_dir}" \
+    --xml_output_file="${out_dir}/sponge_log.xml" \
+    |& tee "${out_dir}/sponge_log.log"
 }
 
 #######################################