Merge pull request #27086 from tkashem/sa-test-fix

[rebase 1.24] disable tests with time limit to facilitate rebase

cmd/openshift-tests/e2e.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"regexp"
 	"strings"
 	"time"
 
@@ -14,7 +15,41 @@ import (
 )
 
 func isDisabled(name string) bool {
-	return strings.Contains(name, "[Disabled")
+	if strings.Contains(name, "[Disabled") {
+		return true
+	}
+
+	return shouldSkipUntil(name)
+}
+
+// shouldSkipUntil allows a test to be skipped with a time limit.
+// the test should be annotated with the 'SkippedUntil' tag, as shown below.
+//   [SkippedUntil:05092022:blocker-bz/123456]
+// - the specified date should conform to the 'MMDDYYYY' format.
+// - a valid blocker BZ must be specified
+// if the specified date in the tag has not passed yet, the test
+// will be skipped by the runner.
+func shouldSkipUntil(name string) bool {
+	re, err := regexp.Compile(`\[SkippedUntil:(\d{8}):blocker-bz\/([a-zA-Z0-9]+)\]`)
+	if err != nil {
+		// it should only happen with a programmer error and unit
+		// test will prevent that
+		return false
+	}
+	matches := re.FindStringSubmatch(name)
+	if len(matches) != 3 {
+		return false
+	}
+
+	skipUntil, err := time.Parse("01022006", matches[1])
+	if err != nil {
+		return false
+	}
+
+	if skipUntil.After(time.Now()) {
+		return true
+	}
+	return false
 }
 
 type testSuite struct {

cmd/openshift-tests/e2e_test.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"fmt"
+	"testing"
+	"time"
+)
+
+func TestSkippedUntil(t *testing.T) {
+	future := func() string { return time.Now().AddDate(0, 0, 5).Format("01022006") }
+	past := func() string { return time.Now().AddDate(0, 0, -5).Format("01022006") }
+	today := func() string { return time.Now().Format("01022006") }
+	unrecognized := func() string { return time.Now().Format("01-02-2006") }
+
+	tests := []struct {
+		name     string
+		testName string
+		skipped  bool
+	}{
+		{
+			name:     "no skipped until tag",
+			testName: "[sig-api-machinery][Feature:APIServer] testing foo [Suite:openshift/...]",
+			skipped:  false,
+		},
+		{
+			name:     "skipped until tag does not specify date or bz",
+			testName: "[sig-api-machinery] testing foo [SkippedUntil:] [Suite:openshift/conformance/parallel/minimal]",
+			skipped:  false,
+		},
+		{
+			name:     "skipped until tag has invalid date",
+			testName: "[sig-api-machinery] testing foo [SkippedUntil:abcdefgh:blocker-bz/123456] [Suite:openshift/conformance/parallel/minimal]",
+			skipped:  false,
+		},
+		{
+			name:     "skipped until tag has unrecognized date format",
+			testName: fmt.Sprintf("[sig-api-machinery] testing foo [SkippedUntil:%s:blocker-bz/123456] [Suite:openshift/...", unrecognized()),
+			skipped:  false,
+		},
+		{
+			name:     "skipped until with valid date in the past",
+			testName: fmt.Sprintf("[sig-api-machinery] testing foo [SkippedUntil:%s:blocker-bz/123456] [Suite:openshift/...]", past()),
+			skipped:  false,
+		},
+		{
+			name:     "skipped until with today's date",
+			testName: fmt.Sprintf("[sig-api-machinery] testing foo [SkippedUntil:%s:blocker-bz/123456] [Suite:openshift/...]", today()),
+			skipped:  false,
+		},
+		{
+			name:     "skipped until with a valid date in the future, but no blocker bz specified",
+			testName: fmt.Sprintf("[sig-api-machinery] testing foo [SkippedUntil:%s:blocker-bz/] [Suite:openshift]", future()),
+			skipped:  false,
+		},
+		{
+			name:     "skipped until with a valid date in the future, blocker bz is malformed",
+			testName: fmt.Sprintf("[sig-api-machinery] testing foo [SkippedUntil:%s:blocker-bz/123*] [Suite:openshift]", future()),
+			skipped:  false,
+		},
+		{
+			name:     "skipped until with a valid date in the future",
+			testName: fmt.Sprintf("[sig-api-machinery] testing foo [SkippedUntil:%s:blocker-bz/123456] [Suite:openshift]", future()),
+			skipped:  true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Logf("test name: %s", test.testName)
+			got := shouldSkipUntil(test.testName)
+			if test.skipped != got {
+				t.Errorf("Expected: %v, but got: %v", test.skipped, got)
+			}
+		})
+	}
+}

test/extended/util/annotate/rules.go

@@ -329,6 +329,22 @@ var (
 			// https://bugzilla.redhat.com/show_bug.cgi?id=1825027
 			`\[Feature:Platform\] Managed cluster should ensure control plane operators do not make themselves unevictable`,
 		},
+
+		// TODO: to facilitate v.14 rebase, skip the following tests until May 27 2022,
+		//  the following key should be removed after the rebase PR lands
+		//  BZs to keep track of these issues:
+		//   - [sig-api-machinery] API data in etcd should be: https://bugzilla.redhat.com/show_bug.cgi?id=2081021
+		//   - [sig-instrumentation] Events API should ensure that: https://bugzilla.redhat.com/show_bug.cgi?id=2081084
+		//   - [sig-auth] ServiceAccounts : https//bugzilla.redhat.com/show_bug.cgi?id=2081087
+		"[SkippedUntil:05272022:blocker-bz/2081087]": {
+			`\[sig-auth\] ServiceAccounts should allow opting out of API token automount`,
+		},
+		"[SkippedUntil:05272022:blocker-bz/2081084]": {
+			`\[sig-instrumentation\] Events API should ensure that an event can be fetched, patched, deleted, and listed`,
+		},
+		"[SkippedUntil:05272022:blocker-bz/2081021]": {
+			`\[sig-api-machinery\] API data in etcd should be stored at the correct location and version for all resources`,
+		},
 	}
 
 	// labelExcludes temporarily block tests out of a specific suite