ci(jenkins): add resilience when timeout issues (#739)

Jenkinsfile

@@ -275,18 +275,20 @@ def generateStep(version){
   return {
     node('linux && immutable'){
       try {
-        deleteDir()
-        unstash 'source'
         echo "${version}"
-        dir("${BASE_DIR}"){
-          withEnv(["GO_VERSION=${version}"]) {
-            // Another retry in case there are any environmental issues
-            // See https://issuetracker.google.com/issues/146072599 for more context
-            retry(2) {
-              sleep randomNumber(min: 2, max: 5)
+        withEnv(["GO_VERSION=${version}"]) {
+          // Another retry in case there are any environmental issues
+          // See https://issuetracker.google.com/issues/146072599 for more context
+          retry(3) {
+            deleteDir()
+            unstash 'source'
+            dir("${BASE_DIR}"){
               sh script: './scripts/jenkins/before_install.sh', label: 'Install dependencies'
+              sh script: './scripts/jenkins/build.sh', label: 'Build'
             }
-            sh script: './scripts/jenkins/build-test.sh', label: 'Build and test'
+          }
+          dir("${BASE_DIR}"){
+            sh script: './scripts/jenkins/test.sh', label: 'Test'
           }
         }
       } catch(e){

scripts/jenkins/build-test.sh

@@ -1,20 +1,5 @@
 #!/usr/bin/env bash
 set -euxo pipefail
 
-# Install Go using the same travis approach
-echo "Installing ${GO_VERSION} with gimme."
-eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=${GO_VERSION} bash)"
-
-make install precheck check-modules
-
-# Run the tests
-set +e
-export OUT_FILE="build/test-report.out"
-mkdir -p build
-make test 2>&1 | tee ${OUT_FILE}
-status=$?
-
-go get -v -u github.com/jstemmer/go-junit-report
-go-junit-report > "build/junit-apm-agent-go-${GO_VERSION}.xml" < ${OUT_FILE}
-
-exit ${status}
+./scripts/jenkins/build.sh
+./scripts/jenkins/test.sh

scripts/jenkins/build.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+# Install Go using the same travis approach
+echo "Installing ${GO_VERSION} with gimme."
+eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=${GO_VERSION} bash)"
+
+make install precheck check-modules

scripts/jenkins/test.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+# Install Go using the same travis approach
+echo "Installing ${GO_VERSION} with gimme."
+eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=${GO_VERSION} bash)"
+
+# Run the tests
+set +e
+export OUT_FILE="build/test-report.out"
+mkdir -p build
+make test 2>&1 | tee ${OUT_FILE}
+status=$?
+
+go get -v -u github.com/jstemmer/go-junit-report
+go-junit-report > "build/junit-apm-agent-go-${GO_VERSION}.xml" < ${OUT_FILE}
+
+exit ${status}