diff --git a/cmd/branchingconfigmanagers/generated-release-gating-jobs/.gitignore b/cmd/branchingconfigmanagers/generated-release-gating-jobs/.gitignore new file mode 100644 index 00000000000..daa6abaaa5b --- /dev/null +++ b/cmd/branchingconfigmanagers/generated-release-gating-jobs/.gitignore @@ -0,0 +1 @@ +generated-release-gating-jobs \ No newline at end of file diff --git a/cmd/branchingconfigmanagers/generated-release-gating-jobs/README.md b/cmd/branchingconfigmanagers/generated-release-gating-jobs/README.md new file mode 100644 index 00000000000..0168243c4a3 --- /dev/null +++ b/cmd/branchingconfigmanagers/generated-release-gating-jobs/README.md @@ -0,0 +1,14 @@ +# Generated release gating jobs manager +This manager attempts to automatize step (2.) of "[Few weeks before branching day](https://docs.google.com/document/d/1Z6ejnDCOCvNv9PWkyNPzVbjuLbDMAAT5GEeDpzb0SMs/edit#heading=h.r9xn02r1cyfn)" phase. + +## Usage +### Options: +- `--current-release` specifies the current OCP version +- `--release-repo` is the absolution path to `openshift/release` repository + +### Example +```sh + $ ./generated-release-gating-jobs \ + --current-release "4.12" \ + --release-repo "/full/path/to/openshift/release/repo" +``` \ No newline at end of file diff --git a/cmd/branchingconfigmanagers/generated-release-gating-jobs/main.go b/cmd/branchingconfigmanagers/generated-release-gating-jobs/main.go new file mode 100644 index 00000000000..7203a5bc034 --- /dev/null +++ b/cmd/branchingconfigmanagers/generated-release-gating-jobs/main.go @@ -0,0 +1,84 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "path" + "time" + + "github.com/sirupsen/logrus" + + utilerrors "k8s.io/apimachinery/pkg/util/errors" + + "github.com/openshift/ci-tools/pkg/api/ocplifecycle" + "github.com/openshift/ci-tools/pkg/branchcuts/bumper" + cioperatorcfg "github.com/openshift/ci-tools/pkg/config" +) + +const ( + releaseJobsPath = "ci-operator/config/openshift/release" +) + +type options struct { + curOCPVersion string + releaseRepoDir string + logLevel int + newIntervalValue int +} + +func gatherOptions() (*options, error) { + var errs []error + o := &options{} + flag.StringVar(&o.curOCPVersion, "current-release", "", "Current OCP version") + flag.StringVar(&o.releaseRepoDir, "release-repo", "", "Path to 'openshift/release/ folder") + flag.IntVar(&o.newIntervalValue, "interval", 168, "New interval to set") + flag.IntVar(&o.logLevel, "log-level", int(logrus.DebugLevel), "Log level") + flag.Parse() + + if _, err := ocplifecycle.ParseMajorMinor(o.curOCPVersion); o.curOCPVersion != "" && err != nil { + errs = append(errs, fmt.Errorf("error parsing current-release %s", o.curOCPVersion)) + } + + if o.newIntervalValue < 0 { + errs = append(errs, errors.New("error parsing interval: value is not a positive integer")) + } + + if o.releaseRepoDir != "" { + if !path.IsAbs(o.releaseRepoDir) { + errs = append(errs, errors.New("error parsing release repo path: path has to be absolute")) + } + } else { + errs = append(errs, errors.New("error parsing release repo path: path is mandatory")) + } + + return o, utilerrors.NewAggregate(errs) +} + +func main() { + o, err := gatherOptions() + if err != nil { + logrus.WithError(err).Fatal("failed to gather options") + } + + logrus.SetLevel(logrus.Level(o.logLevel)) + logrus.Debugf("using options %+v", o) + + if err := reconcile(time.Now(), o); err != nil { + logrus.WithError(err).Fatal("failed to reconcile the status") + } + logrus.Info("status reconciled") +} + +func reconcile(now time.Time, o *options) error { + logrus.Debugf("using options %+v", o) + releaseJobsDir := path.Join(o.releaseRepoDir, releaseJobsPath) + b, err := bumper.NewGeneratedReleaseGatingJobsBumper(o.curOCPVersion, releaseJobsDir, o.newIntervalValue) + if err != nil { + return fmt.Errorf("new bumper: %w", err) + } + if err := bumper.Bump[*cioperatorcfg.DataWithInfo](b, &bumper.BumpingOptions{}); err != nil { + return fmt.Errorf("bumper: %w", err) + } + return nil +} diff --git a/pkg/branchcuts/bumper/gen-release-jobs-bumper.go b/pkg/branchcuts/bumper/gen-release-jobs-bumper.go new file mode 100644 index 00000000000..0c75785741b --- /dev/null +++ b/pkg/branchcuts/bumper/gen-release-jobs-bumper.go @@ -0,0 +1,241 @@ +package bumper + +import ( + "fmt" + "io/fs" + "path" + "path/filepath" + "regexp" + "strconv" + "strings" + + "github.com/sirupsen/logrus" + + cioperatorapi "github.com/openshift/ci-tools/pkg/api" + "github.com/openshift/ci-tools/pkg/api/ocplifecycle" + cioperatorcfg "github.com/openshift/ci-tools/pkg/config" +) + +const ( + releaseJobsRegexPatternFormat = `openshift-release-master__(okd|ci|nightly|okd-scos)-%s.*\.yaml` + ocpReleaseEnvVarName = "OCP_VERSION" +) + +type GeneratedReleaseGatingJobsBumper struct { + mm *ocplifecycle.MajorMinor + getFilesRegexp *regexp.Regexp + jobsDir string + newIntervalValue int +} + +var _ Bumper[*cioperatorcfg.DataWithInfo] = &GeneratedReleaseGatingJobsBumper{} + +func NewGeneratedReleaseGatingJobsBumper(ocpVer, jobsDir string, newIntervalValue int) (*GeneratedReleaseGatingJobsBumper, error) { + mm, err := ocplifecycle.ParseMajorMinor(ocpVer) + if err != nil { + return nil, fmt.Errorf("parse release: %w", err) + } + mmRegexp := fmt.Sprintf("%d\\.%d", mm.Major, mm.Minor) + getFilesRegexp := regexp.MustCompile(fmt.Sprintf(releaseJobsRegexPatternFormat, mmRegexp)) + return &GeneratedReleaseGatingJobsBumper{ + mm, + getFilesRegexp, + jobsDir, + newIntervalValue, + }, nil +} + +func (b *GeneratedReleaseGatingJobsBumper) GetFiles() ([]string, error) { + files := make([]string, 0) + err := filepath.Walk(b.jobsDir, func(path string, info fs.FileInfo, err error) error { + if b.getFilesRegexp.Match([]byte(path)) { + files = append(files, path) + } + return nil + }) + return files, err +} + +func (b *GeneratedReleaseGatingJobsBumper) Unmarshall(file string) (*cioperatorcfg.DataWithInfo, error) { + cfgDataByFilename, err := cioperatorcfg.LoadDataByFilename(file) + if err != nil { + return nil, err + } + + filename := path.Base(file) + dataWithInfo, ok := cfgDataByFilename[filename] + if !ok { + logrus.WithError(err).Errorf("failed to get config %s", file) + return nil, fmt.Errorf("can't get data %s", filename) + } + return &dataWithInfo, nil +} + +func (b *GeneratedReleaseGatingJobsBumper) BumpFilename( + filename string, + dataWithInfo *cioperatorcfg.DataWithInfo) (string, error) { + newVariant, err := ReplaceWithNextVersion(dataWithInfo.Info.Metadata.Variant, b.mm.Major) + if err != nil { + return "", err + } + dataWithInfo.Info.Metadata.Variant = newVariant + return dataWithInfo.Info.Metadata.Basename(), nil +} + +/* + Candidate bumping fields: + .base_images.*.name + .releases.*.{release,candidate}.version + .releases.*.prerelease.version_bounds.{lower,upper} + .tests[].steps.test[].env[].default +*/ +func (b *GeneratedReleaseGatingJobsBumper) BumpContent(dataWithInfo *cioperatorcfg.DataWithInfo) (*cioperatorcfg.DataWithInfo, error) { + major := b.mm.Major + config := &dataWithInfo.Configuration + if err := bumpBaseImages(config, major); err != nil { + return nil, err + } + + if err := bumpReleases(config, major); err != nil { + return nil, err + } + + if err := bumpTests(config, major); err != nil { + return nil, err + } + + if err := ReplaceWithNextVersionInPlace(&config.Metadata.Variant, major); err != nil { + return nil, err + } + + if config.Tests != nil { + for i := 0; i < len(config.Tests); i++ { + if config.Tests[i].Interval != nil { + *config.Tests[i].Interval = strconv.Itoa(b.newIntervalValue) + "h" + } + } + } + + return dataWithInfo, nil +} + +func bumpBaseImages(config *cioperatorapi.ReleaseBuildConfiguration, major int) error { + if config.BaseImages == nil { + return nil + } + + bumpedImages := make(map[string]cioperatorapi.ImageStreamTagReference) + for k := range config.BaseImages { + image := config.BaseImages[k] + if err := ReplaceWithNextVersionInPlace(&image.Name, major); err != nil { + return err + } + bumpedImages[k] = image + } + config.BaseImages = bumpedImages + return nil +} + +func bumpReleases(config *cioperatorapi.ReleaseBuildConfiguration, major int) error { + if config.Releases == nil { + return nil + } + + bumpedReleases := make(map[string]cioperatorapi.UnresolvedRelease) + for k := range config.Releases { + release := config.Releases[k] + if release.Release != nil { + if err := ReplaceWithNextVersionInPlace(&release.Release.Version, major); err != nil { + return err + } + } + if release.Candidate != nil { + if err := ReplaceWithNextVersionInPlace(&release.Candidate.Version, major); err != nil { + return err + } + } + if release.Prerelease != nil { + if err := ReplaceWithNextVersionInPlace(&release.Prerelease.VersionBounds.Upper, major); err != nil { + return err + } + if err := ReplaceWithNextVersionInPlace(&release.Prerelease.VersionBounds.Lower, major); err != nil { + return err + } + } + bumpedReleases[k] = release + } + config.Releases = bumpedReleases + return nil +} + +func bumpTests(config *cioperatorapi.ReleaseBuildConfiguration, major int) error { + if config.Tests == nil { + return nil + } + + for i := 0; i < len(config.Tests); i++ { + test := config.Tests[i] + + if test.MultiStageTestConfiguration == nil { + continue + } + + if err := bumpTestSteps(test.MultiStageTestConfiguration.Pre, major); err != nil { + return err + } + if err := bumpTestSteps(test.MultiStageTestConfiguration.Test, major); err != nil { + return err + } + if err := bumpTestSteps(test.MultiStageTestConfiguration.Post, major); err != nil { + return err + } + + if err := ReplaceWithNextVersionInPlace(test.MultiStageTestConfiguration.Workflow, major); err != nil { + return err + } + + config.Tests[i] = test + } + + return nil +} + +func bumpTestSteps(tests []cioperatorapi.TestStep, major int) error { + if tests == nil { + return nil + } + for i := 0; i < len(tests); i++ { + multistageTest := tests[i] + if err := bumpTestStepEnvVars(multistageTest, major); err != nil { + return err + } + tests[i] = multistageTest + } + return nil +} + +func bumpTestStepEnvVars(multistageTest cioperatorapi.TestStep, major int) error { + if multistageTest.LiteralTestStep == nil || multistageTest.LiteralTestStep.Environment == nil { + return nil + } + for i := 0; i < len(multistageTest.Environment); i++ { + env := multistageTest.Environment[i] + if env.Name == ocpReleaseEnvVarName { + if err := ReplaceWithNextVersionInPlace(env.Default, major); err != nil { + return err + } + } + multistageTest.Environment[i] = env + } + return nil +} + +func (b *GeneratedReleaseGatingJobsBumper) Marshall(dataWithInfo *cioperatorcfg.DataWithInfo, bumpedFilename, dir string) error { + absolutePath := path.Join(dir, bumpedFilename) + dirNoOrgRepo := strings.TrimSuffix(absolutePath, dataWithInfo.Info.Metadata.RelativePath()) + if err := dataWithInfo.CommitTo(dirNoOrgRepo); err != nil { + logrus.WithError(err).Errorf("error saving config %s", dirNoOrgRepo) + return fmt.Errorf("commit to %s failed: %w", dirNoOrgRepo, err) + } + return nil +} diff --git a/pkg/branchcuts/bumper/gen-release-jobs-bumper_test.go b/pkg/branchcuts/bumper/gen-release-jobs-bumper_test.go new file mode 100644 index 00000000000..0b46a0b75ca --- /dev/null +++ b/pkg/branchcuts/bumper/gen-release-jobs-bumper_test.go @@ -0,0 +1,203 @@ +package bumper + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + + cioperatorapi "github.com/openshift/ci-tools/pkg/api" + cioperatorcfg "github.com/openshift/ci-tools/pkg/config" +) + +func TestBumpFilename(t *testing.T) { + tests := []struct { + id string + ocpRelease string + meta cioperatorapi.Metadata + wantFilename string + }{ + { + id: "Bump filename properly", + ocpRelease: "4.11", + meta: cioperatorapi.Metadata{ + Org: "org", + Repo: "repo", + Branch: "br", + Variant: "nightly-4.10", + }, + wantFilename: "org-repo-br__nightly-4.11.yaml", + }, + { + id: "Bump nothing", + ocpRelease: "4.11", + meta: cioperatorapi.Metadata{ + Org: "org", + Repo: "repo", + Branch: "br", + Variant: "nightly-5", + }, + wantFilename: "org-repo-br__nightly-5.yaml", + }, + } + for _, test := range tests { + t.Run(test.id, func(t *testing.T) { + b, err := NewGeneratedReleaseGatingJobsBumper(test.ocpRelease, "", 1) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + dataWithInfo := cioperatorcfg.DataWithInfo{ + Info: cioperatorcfg.Info{ + Metadata: test.meta, + }, + } + filename, err := b.BumpFilename("", &dataWithInfo) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if test.wantFilename != filename { + t.Errorf("filenames are different: %s", cmp.Diff(test.wantFilename, filename)) + } + }) + } +} + +func TestBumpObject(t *testing.T) { + strRef := func(i string) *string { + return &i + } + tests := []struct { + id string + ocpRelease string + interval int + config cioperatorapi.ReleaseBuildConfiguration + wantConfig cioperatorapi.ReleaseBuildConfiguration + }{ + { + id: "Object bumped properly", + ocpRelease: "4.11", + interval: 168, + config: cioperatorapi.ReleaseBuildConfiguration{ + Metadata: cioperatorapi.Metadata{ + Variant: "4.10", + }, + InputConfiguration: cioperatorapi.InputConfiguration{ + BaseImages: map[string]cioperatorapi.ImageStreamTagReference{ + "image-1": { + Name: "image_4.10", + }, + }, + Releases: map[string]cioperatorapi.UnresolvedRelease{ + "release": { + Release: &cioperatorapi.Release{ + Version: "4.10", + }, + }, + "candidate": { + Release: &cioperatorapi.Release{ + Version: "4.10", + }, + }, + "prerelease": { + Prerelease: &cioperatorapi.Prerelease{ + VersionBounds: cioperatorapi.VersionBounds{ + Lower: "4.9", + Upper: "4.11", + }, + }, + }, + }, + }, + Tests: []cioperatorapi.TestStepConfiguration{ + { + Interval: strRef("24h"), + MultiStageTestConfiguration: &cioperatorapi.MultiStageTestConfiguration{ + Workflow: strRef("w-4.10-4.11"), + Test: []cioperatorapi.TestStep{ + { + LiteralTestStep: &cioperatorapi.LiteralTestStep{ + Environment: []cioperatorapi.StepParameter{ + { + Name: "OCP_VERSION", + Default: strRef("4.10"), + }, + }, + }, + }, + }, + }, + }, + }, + }, + wantConfig: cioperatorapi.ReleaseBuildConfiguration{ + Metadata: cioperatorapi.Metadata{ + Variant: "4.11", + }, + InputConfiguration: cioperatorapi.InputConfiguration{ + BaseImages: map[string]cioperatorapi.ImageStreamTagReference{ + "image-1": { + Name: "image_4.11", + }, + }, + Releases: map[string]cioperatorapi.UnresolvedRelease{ + "release": { + Release: &cioperatorapi.Release{ + Version: "4.11", + }, + }, + "candidate": { + Release: &cioperatorapi.Release{ + Version: "4.11", + }, + }, + "prerelease": { + Prerelease: &cioperatorapi.Prerelease{ + VersionBounds: cioperatorapi.VersionBounds{ + Lower: "4.10", + Upper: "4.12", + }, + }, + }, + }, + }, + Tests: []cioperatorapi.TestStepConfiguration{ + { + Interval: strRef("168h"), + MultiStageTestConfiguration: &cioperatorapi.MultiStageTestConfiguration{ + Workflow: strRef("w-4.11-4.12"), + Test: []cioperatorapi.TestStep{ + { + LiteralTestStep: &cioperatorapi.LiteralTestStep{ + Environment: []cioperatorapi.StepParameter{ + { + Name: "OCP_VERSION", + Default: strRef("4.11"), + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + for _, test := range tests { + t.Run(test.id, func(t *testing.T) { + b, err := NewGeneratedReleaseGatingJobsBumper(test.ocpRelease, "", test.interval) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + dataWithInfo := cioperatorcfg.DataWithInfo{ + Configuration: test.config, + } + result, err := b.BumpContent(&dataWithInfo) + if err != nil { + t.Errorf("Unexpected error: %s", err.Error()) + } + if diff := cmp.Diff(&test.wantConfig, &result.Configuration); diff != "" { + t.Errorf("Configurations are different: %s", diff) + } + }) + } +} diff --git a/pkg/branchcuts/bumper/replace.go b/pkg/branchcuts/bumper/replace.go new file mode 100644 index 00000000000..d9ed51bab5f --- /dev/null +++ b/pkg/branchcuts/bumper/replace.go @@ -0,0 +1,70 @@ +package bumper + +import ( + "fmt" + "regexp" + "strconv" + "strings" + + "k8s.io/apimachinery/pkg/util/sets" +) + +func ReplaceWithNextVersionInPlace(line *string, major int) error { + newLine, err := ReplaceWithNextVersion(*line, major) + if err != nil { + return err + } + *line = newLine + return nil +} + +// Find every {major}.{minor} reference into 'line' and the replace it +// with {major}.{minor+1} +func ReplaceWithNextVersion(line string, major int) (string, error) { + p := fmt.Sprintf(`%d\.(?P\d+)`, major) + r := regexp.MustCompile(p) + m := r.FindAllStringSubmatch(line, -1) + if m == nil { + return line, nil + } + + minors, err := uniqueSortedMinors(m) + if err != nil { + return line, err + } + + for i := len(minors) - 1; i >= 0; i-- { + minor := minors[i] + curVersion := fmt.Sprintf("%d.%d", major, minor) + nextVersion := fmt.Sprintf("%d.%d", major, minor+1) + line = strings.ReplaceAll(line, curVersion, nextVersion) + } + + return line, nil +} + +// The function extracts all the minors from matches, removes duplicate and finally it returs them in +// increasing order. +// +// Matches is an array of regex matches that could be obtained by the following example: +// string: ocp_4.5-4.6-4.5 +// pattern: 4\.\d+ +// matches: [3]string{} +// matches[0]: []string{ "4.5", "5" } +// matches[1]: []string{ "4.6", "6" } +// matches[2]: []string{ "4.5", "5" } +// +// Given the previous input, the function returns []int{5, 6} +func uniqueSortedMinors(matches [][]string) ([]int, error) { + minors := sets.NewInt() + for _, m := range matches { + for i := 1; i < len(m); i++ { + minor, err := strconv.Atoi(m[i]) + if err != nil { + return []int{}, err + } + minors.Insert(minor) + } + } + return minors.List(), nil +} diff --git a/pkg/branchcuts/bumper/replace_test.go b/pkg/branchcuts/bumper/replace_test.go new file mode 100644 index 00000000000..aa28275ff9b --- /dev/null +++ b/pkg/branchcuts/bumper/replace_test.go @@ -0,0 +1,70 @@ +package bumper_test + +import ( + "testing" + + "github.com/openshift/ci-tools/pkg/branchcuts/bumper" +) + +func TestReplaceWithNextVersion(t *testing.T) { + tests := []struct { + name string + line string + major int + expected string + }{ + { + name: "Bump to the next version properly", + line: "product_3.2", + major: 3, + expected: "product_3.3", + }, + { + name: "Bump skipped due to major mismatch", + line: "product_3.2", + major: 2, + expected: "product_3.2", + }, + { + name: "Unable to bump when leading zeroes", + line: "product_3.002", + major: 3, + expected: "product_3.002", + }, + { + name: "Multiple bumping", + line: "product_3.2 product_3.9", + major: 3, + expected: "product_3.3 product_3.10", + }, + { + name: "Multiple bumping 2", + line: "openshift-upgrade-ovirt-release-4.5-4.6", + major: 4, + expected: "openshift-upgrade-ovirt-release-4.6-4.7", + }, + { + name: "Multiple bumping with a major mismatch", + line: "product_3.2 product_3.9 product_4.1", + major: 3, + expected: "product_3.3 product_3.10 product_4.1", + }, + { + name: "Unexpected dot", + line: "product_3..2", + major: 3, + expected: "product_3..2", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + l, err := bumper.ReplaceWithNextVersion(test.line, test.major) + if err != nil { + t.Error(err) + } else if l != test.expected { + t.Errorf("Expected %s but got %s", test.expected, l) + } + }) + } +}