Merge pull request #6988 from taskcluster/matt-boris/genericWorkerGra…

…cefulTerminationRaceFix

fix(generic-worker): check for graceful termination before claiming work

changelog/issue-6987.md

@@ -0,0 +1,7 @@
+audience: users
+level: patch
+reference: issue 6987
+---
+Generic Worker now checks if a graceful termination was requested from worker runner _before_ calling `queue.claimWork()`.
+
+This helps fix a race condition where a preemption occurs right after Generic Worker starts up, but before the graceful termination handler to abort the task has been initialized.

workers/generic-worker/main.go

@@ -441,6 +441,10 @@ func RunWorker() (exitCode ExitCode) {
 			panic(err)
 		}
 
+		if graceful.TerminationRequested() {
+			return WORKER_SHUTDOWN
+		}
+
 		task := ClaimWork()
 
 		// make sure at least 5 seconds pass between tcqueue.ClaimWork API calls
@@ -518,10 +522,6 @@ func RunWorker() (exitCode ExitCode) {
 			}
 		}
 
-		if graceful.TerminationRequested() {
-			return WORKER_SHUTDOWN
-		}
-
 		// To avoid hammering queue, make sure there is at least 5 seconds
 		// between consecutive requests. Note we do this even if a task ran,
 		// since a task could complete in less than that amount of time.