/
continuous.sh
executable file
·176 lines (155 loc) · 6.77 KB
/
continuous.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
#!/bin/bash
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##
# continuous.sh
# Runs CI checks for entire repository.
#
# Jobs types
#
# Continuous: Runs root tests & tests in submodules changed by a PR. Triggered by PR merges.
# Nightly: Runs root tests & tests in all modules. Triggered nightly.
# Nightly/$MODULE: Runs tests in a specified module. Triggered nightly.
##
export GOOGLE_APPLICATION_CREDENTIALS=$KOKORO_KEYSTORE_DIR/72523_go_integration_service_account
# Removing the GCLOUD_TESTS_GOLANG_PROJECT_ID setting may make some integration
# tests (like profiler's) silently skipped, so make sure you know what you are
# doing when changing / removing the next line.
export GCLOUD_TESTS_GOLANG_PROJECT_ID=dulcet-port-762
export GCLOUD_TESTS_GOLANG_KEY=$GOOGLE_APPLICATION_CREDENTIALS
export GCLOUD_TESTS_GOLANG_FIRESTORE_PROJECT_ID=gcloud-golang-firestore-tests
export GCLOUD_TESTS_GOLANG_FIRESTORE_KEY=$KOKORO_KEYSTORE_DIR/72523_go_firestore_integration_service_account
export GCLOUD_TESTS_API_KEY=`cat $KOKORO_KEYSTORE_DIR/72523_go_gcloud_tests_api_key`
export GCLOUD_TESTS_GOLANG_KEYRING=projects/dulcet-port-762/locations/us/keyRings/go-integration-test
export GCLOUD_TESTS_GOLANG_PROFILER_ZONE="us-west1-b"
# Bigtable integration tests expect an existing instance and cluster
# ❯ cbt createinstance gc-bt-it-instance gc-bt-it-instance \
# gc-bt-it-cluster us-west1-b 1 SSD
export GCLOUD_TESTS_BIGTABLE_KEYRING=projects/dulcet-port-762/locations/us-central1/keyRings/go-integration-test-regional
export GCLOUD_TESTS_BIGTABLE_CLUSTER="gc-bt-it-cluster"
export GCLOUD_TESTS_BIGTABLE_INSTANCE="gc-bt-it-instance"
# TODO: Remove this env after OMG/43748 is fixed
# Spanner integration tests for backup/restore is flaky https://github.com/googleapis/google-cloud-go/issues/5037
# to fix the flaky test Spanner need to run on us-west1 region.
export GCLOUD_TESTS_GOLANG_SPANNER_INSTANCE_CONFIG="regional-us-west1"
# Fail on any error
set -eo pipefail
# Display commands being run
set -x
# cd to project dir on Kokoro instance
# TODO(#6444): change this back to github/google-cloud-go before merging to main
cd github/google-cloud-go-storage-refactor
go version
export GOCLOUD_HOME=$KOKORO_ARTIFACTS_DIR/google-cloud-go/
export PATH="$GOPATH/bin:$PATH"
export GO111MODULE=on
export GOPROXY=https://proxy.golang.org
# Move code into artifacts dir
mkdir -p $GOCLOUD_HOME
git clone . $GOCLOUD_HOME
cd $GOCLOUD_HOME
try3() { eval "$*" || eval "$*" || eval "$*"; }
# All packages, including +build tools, are fetched.
try3 go mod download
./internal/kokoro/vet.sh
# runDirectoryTests runs all tests in the current directory.
# If a PATH argument is specified, it runs `go test [PATH]`.
runDirectoryTests() {
go test -race -v -timeout 45m "${1:-./...}" 2>&1 \
| tee sponge_log.log
# Takes the kokoro output log (raw stdout) and creates a machine-parseable
# xUnit XML file.
cat sponge_log.log \
| go-junit-report -set-exit-code > sponge_log.xml
# Add the exit codes together so we exit non-zero if any module fails.
exit_code=$(($exit_code + $?))
}
# runEmulatorTests runs emulator tests in the current directory.
runEmulatorTests() {
if [ -f "emulator_test.sh" ]; then
./emulator_test.sh
fi
# Takes the kokoro output log (raw stdout) and creates a machine-parseable
# xUnit XML file.
cat sponge_log.log \
| go-junit-report -set-exit-code > sponge_log.xml
# Add the exit codes together so we exit non-zero if any module fails.
exit_code=$(($exit_code + $?))
}
# testAllModules runs all modules' tests, including emulator tests.
testAllModules() {
echo "Testing all modules"
for i in $(find . -name go.mod); do
pushd "$(dirname "$i")" > /dev/null;
runDirectoryTests
# Run integration tests against an emulator.
runEmulatorTests
popd > /dev/null;
done
}
# testChangedModules runs tests in changed modules only.
testChangedModules() {
for d in $CHANGED_DIRS; do
goDirectories="$(find "$d" -name "*.go" -printf "%h\n" | sort -u)"
if [[ -n "$goDirectories" ]]; then
for gd in $goDirectories; do
pushd "$gd" > /dev/null;
runDirectoryTests .
popd > /dev/null;
done
fi
done
}
set +e # Run all tests, don't stop after the first failure.
exit_code=0
if [[ $KOKORO_JOB_NAME == *"continuous"* ]]; then
# Continuous jobs only run root tests & tests in submodules changed by the PR.
SIGNIFICANT_CHANGES=$(git --no-pager diff --name-only $KOKORO_GIT_COMMIT^..$KOKORO_GIT_COMMIT | grep -Ev '(\.md$|^\.github)' || true)
# CHANGED_DIRS is the list of significant top-level directories that changed,
# but weren't deleted by the current PR. CHANGED_DIRS will be empty when run on main.
CHANGED_DIRS=$(echo "$SIGNIFICANT_CHANGES" | tr ' ' '\n' | grep "/" | cut -d/ -f1 | sort -u | tr '\n' ' ' | xargs ls -d 2>/dev/null || true)
# If PR changes affect all submodules, then run all tests.
if [[ -z $SIGNIFICANT_CHANGES ]] || echo "$SIGNIFICANT_CHANGES" | tr ' ' '\n' | grep "^go.mod$" || [[ $CHANGED_DIRS =~ "internal" ]]; then
testAllModules
else
runDirectoryTests . # Always run base tests.
echo "Running tests only in changed submodules: $CHANGED_DIRS"
testChangedModules
fi
elif [[ $KOKORO_JOB_NAME == *"nightly"* ]]; then
# Expected job name format: ".../nightly/[OPTIONAL_MODULE_NAME]/[OPTIONAL_JOB_NAMES...]"
ARR=(${KOKORO_JOB_NAME//// }) # Splits job name by "/" where ARR[0] is expected to be "nightly".
SUBMODULE_NAME=${ARR[5]} # Gets the token after "nightly/".
if [[ -n $SUBMODULE_NAME ]] && [[ -d "./$SUBMODULE_NAME" ]]; then
# Only run tests in the submodule designated in the Kokoro job name.
# Expected format example: ...google-cloud-go/nightly/logging.
runDirectoryTests . # Always run base tests
echo "Running tests in one submodule: $SUBMODULE_NAME"
pushd $SUBMODULE_NAME > /dev/null;
runDirectoryTests
popd > /dev/null
else
# Run all tests if it is a regular nightly job.
testAllModules
fi
else
testAllModules
fi
if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]] || [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"nightly"* ]]; then
chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot
$KOKORO_GFILE_DIR/linux_amd64/flakybot -logs_dir=$GOCLOUD_HOME \
-repo=googleapis/google-cloud-go \
-commit_hash=$KOKORO_GITHUB_COMMIT_URL_google_cloud_go
fi
exit $exit_code