diff --git a/cmd/branchingconfigmanagers/docs/RELEASE-JOBS-COMMON.md b/cmd/branchingconfigmanagers/docs/RELEASE-JOBS-COMMON.md new file mode 100644 index 00000000000..3c6d0ed2aa4 --- /dev/null +++ b/cmd/branchingconfigmanagers/docs/RELEASE-JOBS-COMMON.md @@ -0,0 +1,5 @@ +## Notes no how to specify paths +Every path specified below is intended to be an absolute path: +- BAD: `my/relative/path` +- GOOD: `/my/nice/absolute/path/to/config.yaml` +- ALSO-GOOD: `/my/nice/absolute/path/to/a/dir` diff --git a/cmd/branchingconfigmanagers/generated-release-gating-jobs/.gitignore b/cmd/branchingconfigmanagers/generated-release-gating-jobs/.gitignore new file mode 100644 index 00000000000..daa6abaaa5b --- /dev/null +++ b/cmd/branchingconfigmanagers/generated-release-gating-jobs/.gitignore @@ -0,0 +1 @@ +generated-release-gating-jobs \ No newline at end of file diff --git a/cmd/branchingconfigmanagers/generated-release-gating-jobs/Makefile b/cmd/branchingconfigmanagers/generated-release-gating-jobs/Makefile new file mode 100644 index 00000000000..5d459c1b269 --- /dev/null +++ b/cmd/branchingconfigmanagers/generated-release-gating-jobs/Makefile @@ -0,0 +1,7 @@ +build: + go build ./... +.PHONY: build + +clean: + go clean ./... +.PHONY: clean diff --git a/cmd/branchingconfigmanagers/generated-release-gating-jobs/README.md b/cmd/branchingconfigmanagers/generated-release-gating-jobs/README.md new file mode 100644 index 00000000000..8fe878f9ae5 --- /dev/null +++ b/cmd/branchingconfigmanagers/generated-release-gating-jobs/README.md @@ -0,0 +1,35 @@ +# Generated release gating jobs manager +This manager attempts to automatize step (2.) of "[Few weeks before code freeze](https://docs.google.com/document/d/19kxmzXFnXrbLChZXBfxy68mCYg06j3qT93VF23wepog/edit#heading=h.r9xn02r1cyfn)" phase. + +## Usage +Check [Notes no how to specify paths](../docs/RELEASE-JOBS-COMMON.md#notes-no-how-to-specify-paths) out first. + +The `--lifecycle-config` switch is always mandatory and it has to point to a valid `ocplifecycle.yaml` config file. + +The `--release-repo-path` has to point to the `openshift/release` folder. +When is specified and no others configurations are provided, the manager defaults `--release-gating` dir to `./ci-operator/config/openshift/release` + +The previous path is relative to `--release-repo-path` + +### Example 1 +```sh + OCP_LIFECYCLE_CONFIG_FULLPATH="/full/path/to/ocplifecycle.yaml" + RELEASE_PROJECT_FULLPATH="/full/path/to/release/project" + $ ./generated-release-gating-jobs \ + --lifecycle-config "$OCP_LIFECYCLE_CONFIG_FULLPATH" \ + --release-repo-path "$RELEASE_PROJECT_FULLPATH" +``` + +Alternatively release-gatings jobs path can be overridden: + +### Example 2 +```sh + OCP_LIFECYCLE_CONFIG_FULLPATH="/full/path/to/ocplifecycle.yaml" + RELEASE_GATING_JOBS_DIR_FULLPATH="/full/path/to/release/gating/jobs/dir" + $ ./generated-release-gating-jobs \ + --lifecycle-config "$OCP_LIFECYCLE_CONFIG_FULLPATH" \ + --release-jobs "$RELEASE_GATING_JOBS_DIR_FULLPATH" +``` + +## Makefile +Run `make integration-tests` to launch integration tests. \ No newline at end of file diff --git a/cmd/branchingconfigmanagers/generated-release-gating-jobs/main.go b/cmd/branchingconfigmanagers/generated-release-gating-jobs/main.go new file mode 100644 index 00000000000..8761ca41a64 --- /dev/null +++ b/cmd/branchingconfigmanagers/generated-release-gating-jobs/main.go @@ -0,0 +1,158 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "os" + "path" + "time" + + "github.com/sirupsen/logrus" + + utilerrors "k8s.io/apimachinery/pkg/util/errors" + + "github.com/openshift/ci-tools/pkg/api/ocplifecycle" + "github.com/openshift/ci-tools/pkg/branchcuts/bumper" + mgr "github.com/openshift/ci-tools/pkg/branchcuts/config-manager" + cioperatorcfg "github.com/openshift/ci-tools/pkg/config" +) + +const ( + optsParsingErrorExitStatus = 1 + reconcileErrorExitStatus = 2 + releaseJobsPath = "ci-operator/config/openshift/release" + howManyDaysBefore = 14 + lifecycleConfigFilePathDefaultPath = "" +) + +type options struct { + curOCPVersion string + futureOCPVersion string + releaseJobsDir string + releaseRepoDir string + outDir string + lifecycleConfigFilePath string + logLevel int + newIntervalValue int +} + +func (o *options) tryOverrideOCPVersions(curVersion, futureVersion string) { + if o.curOCPVersion == "" { + o.curOCPVersion = curVersion + } + if o.futureOCPVersion == "" { + o.futureOCPVersion = futureVersion + } +} + +func (o *options) adjustPaths() error { + if o.releaseRepoDir != "" { + if !path.IsAbs(o.releaseRepoDir) { + return errors.New("release-repo-path path has to be absolute") + } + + if path.IsAbs(o.releaseJobsDir) { + return errors.New("release-jobs path can't be absolute when release-jobs is specified") + } + + o.releaseJobsDir = path.Join(o.releaseRepoDir, o.releaseJobsDir) + } + return nil +} + +func gatherOptions() (*options, error) { + var errs []error + o := &options{} + flag.StringVar(&o.curOCPVersion, "current-release", "", "Current OCP version") + flag.StringVar(&o.futureOCPVersion, "future-release", "", "Next OCP version") + flag.StringVar(&o.releaseJobsDir, "release-jobs", releaseJobsPath, "Path to release-gating jobs .yaml files") + flag.StringVar(&o.releaseRepoDir, "release-repo-path", "", "Path to 'openshift/release/ folder") + flag.StringVar(&o.outDir, "out", "", "Output directory. Default to --release-jobs dir") + flag.StringVar(&o.lifecycleConfigFilePath, "lifecycle-config", lifecycleConfigFilePathDefaultPath, "Path to the lifecycle config file") + flag.IntVar(&o.newIntervalValue, "interval", 168, "New interval to set") + flag.IntVar(&o.logLevel, "log-level", int(logrus.DebugLevel), "Log level") + flag.Parse() + + if _, err := ocplifecycle.ParseMajorMinor(o.curOCPVersion); o.curOCPVersion != "" && err != nil { + errs = append(errs, fmt.Errorf("error parsing current-release %s", o.curOCPVersion)) + } + + if _, err := ocplifecycle.ParseMajorMinor(o.futureOCPVersion); o.futureOCPVersion != "" && err != nil { + errs = append(errs, fmt.Errorf("error parsing future-release %s", o.futureOCPVersion)) + } + + if o.newIntervalValue < 0 { + errs = append(errs, errors.New("error parsing interval: value is not a positive integer")) + } + + if o.lifecycleConfigFilePath == "" { + errs = append(errs, errors.New("lifecycle-config is mandatory")) + } + if !path.IsAbs(o.lifecycleConfigFilePath) { + errs = append(errs, errors.New("lifecycle-config path is not absolute")) + } + + if o.outDir != "" && !path.IsAbs(o.outDir) { + errs = append(errs, errors.New("out path is not absolute")) + } + + return o, utilerrors.NewAggregate(errs) +} + +func main() { + o, err := gatherOptions() + if err != nil { + logrus.WithError(err).Error("failed to gather options") + os.Exit(optsParsingErrorExitStatus) + } + + if err = o.adjustPaths(); err != nil { + logrus.WithError(err).Error("failed to gather options") + os.Exit(optsParsingErrorExitStatus) + } + + logrus.SetLevel(logrus.Level(o.logLevel)) + logrus.Debugf("using options %+v", o) + + recStatus, err := reconcile(time.Now(), o) + switch recStatus { + case mgr.ReconcileOk: + logrus.Info("status reconciled") + default: + logrus.WithError(err).Error("failed to reconcile the status") + os.Exit(reconcileErrorExitStatus) + } +} + +func reconcile(now time.Time, o *options) (mgr.ReconcileStatus, error) { + lifecycleConfig, err := ocplifecycle.LoadConfig(o.lifecycleConfigFilePath) + if err != nil { + return mgr.ReconcileErr, fmt.Errorf("OCP Lifecycle load: %w", err) + } + + ocpLifecycle := ocplifecycle.NewOCPCodeFreezeLifecycle(&lifecycleConfig, now) + + if !ocpLifecycle.IsBeforeCodeFreeze(howManyDaysBefore) { + return mgr.ReconcileSkipped, nil + } + + currentVersion, futureVersion, err := ocpLifecycle.GetOCPVersions() + if err != nil { + return mgr.ReconcileErr, fmt.Errorf("OCP Lifecycle versions: %w", err) + } + + o.tryOverrideOCPVersions(currentVersion.GetVersion(), futureVersion.GetVersion()) + logrus.Debugf("using options %+v", o) + + b := bumper.NewGeneratedReleaseGatingJobsBumper(o.curOCPVersion, o.releaseJobsDir, o.newIntervalValue) + bumped, err := bumper.Bump[*cioperatorcfg.DataWithInfo](b, &bumper.BumpingOptions{ + OutDir: o.outDir, + }) + + if !bumped || err != nil { + return mgr.ReconcileErr, fmt.Errorf("bumper: %w", err) + } + + return mgr.ReconcileOk, nil +} diff --git a/cmd/branchingconfigmanagers/rpm-deps-mirroring-services/.gitignore b/cmd/branchingconfigmanagers/rpm-deps-mirroring-services/.gitignore new file mode 100644 index 00000000000..e71ba72024c --- /dev/null +++ b/cmd/branchingconfigmanagers/rpm-deps-mirroring-services/.gitignore @@ -0,0 +1 @@ +rpm-deps-mirroring-services \ No newline at end of file diff --git a/cmd/branchingconfigmanagers/rpm-deps-mirroring-services/Makefile b/cmd/branchingconfigmanagers/rpm-deps-mirroring-services/Makefile new file mode 100644 index 00000000000..5d459c1b269 --- /dev/null +++ b/cmd/branchingconfigmanagers/rpm-deps-mirroring-services/Makefile @@ -0,0 +1,7 @@ +build: + go build ./... +.PHONY: build + +clean: + go clean ./... +.PHONY: clean diff --git a/cmd/branchingconfigmanagers/rpm-deps-mirroring-services/README.md b/cmd/branchingconfigmanagers/rpm-deps-mirroring-services/README.md new file mode 100644 index 00000000000..ab5e5b77489 --- /dev/null +++ b/cmd/branchingconfigmanagers/rpm-deps-mirroring-services/README.md @@ -0,0 +1,35 @@ +# RPM dependencies mirroring services +This manager attempts to automatize step (1.) of "[Few weeks before code freeze](https://docs.google.com/document/d/19kxmzXFnXrbLChZXBfxy68mCYg06j3qT93VF23wepog/edit#heading=h.r9xn02r1cyfn)" phase. + +## Usage +Check [Notes no how to specify paths](../docs/RELEASE-JOBS-COMMON.md#notes-no-how-to-specify-paths) out first. + +The `--lifecycle-config` switch is always mandatory and it has to point to a valid `ocplifecycle.yaml` config file. + +The `--release-repo-path` has to point to the `openshift/release` folder. +If not specified, the manager defaults `--mirroring-services` dir to `core-services/release-controller/_repos` + +The previous paths are relative to `--release-repo-path` + +### Example +```sh + OCP_LIFECYCLE_CONFIG_FULLPATH="/full/path/to/ocplifecycle.yaml" + RELEASE_PROJECT_FULLPATH="/full/path/to/release/project" + $ ./generated-release-gating-jobs \ + --lifecycle-config "$OCP_LIFECYCLE_CONFIG_FULLPATH" \ + --release-repo-path "$RELEASE_PROJECT_FULLPATH" +``` + +Alternatively `--mirroring-services` path can be overridden: + +### Example +```sh + OCP_LIFECYCLE_CONFIG_FULLPATH="/full/path/to/ocplifecycle.yaml" + RPM_DEPS_MIRRORING_SERVICES_FULLPATH="/full/path/to/rpm/deps/mirroring/services" + $ ./generated-release-gating-jobs \ + --lifecycle-config "$OCP_LIFECYCLE_CONFIG_FULLPATH" \ + --mirroring-services "$RPM_DEPS_MIRRORING_SERVICES_FULLPATH" +``` + +## Makefile +Run `make integration-tests` to launch integration tests. \ No newline at end of file diff --git a/cmd/branchingconfigmanagers/rpm-deps-mirroring-services/main.go b/cmd/branchingconfigmanagers/rpm-deps-mirroring-services/main.go new file mode 100644 index 00000000000..bbbcf9fa8d9 --- /dev/null +++ b/cmd/branchingconfigmanagers/rpm-deps-mirroring-services/main.go @@ -0,0 +1,157 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "os" + "path" + "time" + + "github.com/sirupsen/logrus" + "gopkg.in/ini.v1" + + utilerrors "k8s.io/apimachinery/pkg/util/errors" + + "github.com/openshift/ci-tools/pkg/api/ocplifecycle" + "github.com/openshift/ci-tools/pkg/branchcuts/bumper" + mgr "github.com/openshift/ci-tools/pkg/branchcuts/config-manager" +) + +const ( + optsParsingErrorExitStatus = 1 + reconcileErrorExitStatus = 2 + rpmMirroringServicesPath = "core-services/release-controller/_repos" + rpmMirroringServicesGlobPatternFormat = "ocp-%s*.repo" + howManyDaysBefore = 14 + lifecycleConfigFileDefaultPath = "" +) + +type options struct { + curOCPVersion string + releaseRepoDir string + rpmMirroringServicesDir string + outDir string + lifecycleConfigFilePath string + logLevel int +} + +func (o *options) tryOverrideOCPVersion(curVersion string) { + if o.curOCPVersion == "" { + o.curOCPVersion = curVersion + } +} + +func (o *options) adjustPaths() error { + if o.releaseRepoDir != "" { + if !path.IsAbs(o.releaseRepoDir) { + return errors.New("release-repo-path path has to be absolute") + } + + if path.IsAbs(o.rpmMirroringServicesDir) { + return errors.New("mirroring-services path can't be absolute when release-jobs is specified") + } + + o.rpmMirroringServicesDir = path.Join(o.releaseRepoDir, o.rpmMirroringServicesDir) + } + return nil +} + +func gatherOptions() (*options, error) { + var errs []error + o := &options{} + flag.StringVar(&o.curOCPVersion, "current-release", "", "Current OCP version") + flag.StringVar(&o.releaseRepoDir, "release-repo-path", "", "Path to 'openshift/release/ folder") + flag.StringVar(&o.rpmMirroringServicesDir, "mirroring-services", rpmMirroringServicesPath, + "Folder containing RPM mirroring services .repo files") + flag.StringVar(&o.outDir, "out", "", "Output directory. Default to --release-jobs dir") + flag.StringVar(&o.lifecycleConfigFilePath, "lifecycle-config", lifecycleConfigFileDefaultPath, + "Path to the lifecycle config file") + flag.IntVar(&o.logLevel, "log-level", int(logrus.DebugLevel), "Log level") + flag.Parse() + + if _, err := ocplifecycle.ParseMajorMinor(o.curOCPVersion); o.curOCPVersion != "" && err != nil { + errs = append(errs, fmt.Errorf("error parsing cur-ocp-ver %s", o.curOCPVersion)) + } + + if o.lifecycleConfigFilePath == "" { + errs = append(errs, errors.New("lifecycle-config is mandatory")) + } + if !path.IsAbs(o.lifecycleConfigFilePath) { + errs = append(errs, errors.New("lifecycle-config path is not absolute")) + } + + if o.rpmMirroringServicesDir != rpmMirroringServicesPath && !path.IsAbs(o.rpmMirroringServicesDir) { + errs = append(errs, errors.New("mirroring-services path is not absolute")) + } + + if o.outDir != "" && !path.IsAbs(o.outDir) { + errs = append(errs, errors.New("out path is not absolute")) + } + + return o, utilerrors.NewAggregate(errs) +} + +func main() { + o, err := gatherOptions() + if err != nil { + logrus.WithError(err).Error("failed to gather options") + os.Exit(optsParsingErrorExitStatus) + } + + if err = o.adjustPaths(); err != nil { + logrus.WithError(err).Error("failed to gather options") + os.Exit(optsParsingErrorExitStatus) + } + + logrus.SetLevel(logrus.Level(o.logLevel)) + logrus.Debugf("using options %+v", o) + + recStatus, err := reconcile(time.Now(), o) + switch recStatus { + case mgr.ReconcileOk: + logrus.Info("status reconciled") + default: + logrus.WithError(err).Error("failed to reconcile the status") + os.Exit(reconcileErrorExitStatus) + } +} + +func reconcile(now time.Time, o *options) (mgr.ReconcileStatus, error) { + lifecycleConfig, err := ocplifecycle.LoadConfig(o.lifecycleConfigFilePath) + if err != nil { + return mgr.ReconcileErr, fmt.Errorf("OCP Lifecycle load: %w", err) + } + + ocpLifecycle := ocplifecycle.NewOCPCodeFreezeLifecycle(&lifecycleConfig, now) + + if !ocpLifecycle.IsBeforeCodeFreeze(howManyDaysBefore) { + return mgr.ReconcileSkipped, nil + } + + currentVersion, _, err := ocpLifecycle.GetOCPVersions() + if err != nil { + return mgr.ReconcileErr, fmt.Errorf("OCP Lifecycle versions: %w", err) + } + + o.tryOverrideOCPVersion(currentVersion.GetVersion()) + logrus.Debugf("using options %+v", o) + + bumpOpts := bumper.RepoBumperOptions{ + FilesDir: o.rpmMirroringServicesDir, + GlobPattern: fmt.Sprintf(rpmMirroringServicesGlobPatternFormat, o.curOCPVersion), + CurOCPRelease: o.curOCPVersion, + } + logrus.Debugf("bumpOpts: %+v", bumpOpts) + b := bumper.NewRepoBumper(&bumpOpts) + + bumped, err := bumper.Bump[*ini.File](b, &bumper.BumpingOptions{ + OutDir: o.outDir, + }) + + if !bumped || err != nil { + return mgr.ReconcileErr, fmt.Errorf("bumper: %w", err) + } + + return mgr.ReconcileOk, nil +} diff --git a/go.mod b/go.mod index 590dbca97ba..67303e1ee03 100644 --- a/go.mod +++ b/go.mod @@ -196,7 +196,7 @@ require ( google.golang.org/grpc v1.38.0 // indirect google.golang.org/protobuf v1.26.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.62.0 // indirect + gopkg.in/ini.v1 v1.62.0 gopkg.in/square/go-jose.v2 v2.3.1 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b k8s.io/apiextensions-apiserver v0.22.2 // indirect diff --git a/hack/lib/branchcuts/.gitignore b/hack/lib/branchcuts/.gitignore new file mode 100644 index 00000000000..63449f772e5 --- /dev/null +++ b/hack/lib/branchcuts/.gitignore @@ -0,0 +1 @@ +fields.txt \ No newline at end of file diff --git a/hack/lib/branchcuts/Makefile b/hack/lib/branchcuts/Makefile new file mode 100644 index 00000000000..3714695c41a --- /dev/null +++ b/hack/lib/branchcuts/Makefile @@ -0,0 +1,20 @@ +rpm-deps-mirror-srv-ocp-release-field: + files_dir="" + find "${files_dir}" \ + -regextype "posix-extended" -regex '.*ocp-4\.[[:digit:]]+.*\.repo' \ + -exec grep -E '4(\.|-)[[:digit:]]+' "{}" \; \ + > fields.txt +.PHONY: rpm-deps-mirror-srv-ocp-release-field + +rpm-deps-mirror-srv-all-fields: + files_dir="" + find "${files_dir}" -iname "*.repo" -exec cat {} \; \ + | ./print_yum_repo_keys.py +.PHONY: rpm-deps-mirror-srv-all-fields + +gen-release-jobs: + files_dir="" + test ! -z "${files_dir}" || exit 1 + ./search_yaml_fields.py "${files_dir}" '(okd|ci|nightly)-4\.\d+.*\.yaml' '4\.\d+' \ + > fields.txt +.PHONY: gen-release-jobs diff --git a/hack/lib/branchcuts/print_yum_repo_keys.py b/hack/lib/branchcuts/print_yum_repo_keys.py new file mode 100755 index 00000000000..2271a5dea60 --- /dev/null +++ b/hack/lib/branchcuts/print_yum_repo_keys.py @@ -0,0 +1,17 @@ +#!/usr/bin/python + +import re +import sys + +fields=dict() + +for line in sys.stdin: + m = re.match('(.+?)\=(.*)', line) + if m is None: continue + k = m.groups(1)[0] + v = m.groups(1)[1] + fields[k] = v + +for k in sorted(fields.keys()): + v = fields[k] + print(f"{k} = {v}") diff --git a/hack/lib/branchcuts/search_yaml_fields.py b/hack/lib/branchcuts/search_yaml_fields.py new file mode 100755 index 00000000000..002a1151c5c --- /dev/null +++ b/hack/lib/branchcuts/search_yaml_fields.py @@ -0,0 +1,73 @@ +#!/usr/bin/python + +import os +import re +import sys +import yaml +import json + +unmarshall_json = False +base_dir = sys.argv[1] +regex_files = sys.argv[2] +regex_field = sys.argv[3] +if len(sys.argv) > 4: + unmarshall_json = sys.argv[4] == "1" + +def search(obj, path, paths): + if type(obj) == type(""): + v = str(obj) + m = re.search(pattern=regex_field, string=v) + if m is not None: + paths[path] = v + elif type(obj) == dict: + for k in obj: + search(obj[k], f"{path}.{k}", paths) + elif type(obj) == [] or type(obj) == list: + for o in obj: + search(o, f"{path}[]", paths) + +def get_files(base_dir): + for (dirpath, _, filenames) in os.walk(base_dir): + result = [] + for f in filenames: + if re.search(pattern=regex_files, string=f) is not None: + result.append(os.path.join(dirpath, f)) + return result + +def dump(file, paths): + print("****") + print(f"FILE: {os.path.basename(file)}") + for path in sorted(paths.keys()): + print(path) + print("****") + +def merge(d1, d2): + for k in d2: + d1[k] = d2[k] + +def yaml_loader(file: str) -> any: + return yaml.load(file, Loader=yaml.FullLoader) + +def json_loader(file: str) -> any: + return json.load(file) + +loader = yaml_loader +if unmarshall_json: + loader = json_loader + +files = get_files(base_dir) +uniques = dict() + +for f in files: + with open(f, 'r') as config_file: + paths = dict() + parsed_config = loader(config_file) + search(parsed_config, "", paths) + merge(uniques, paths) + if len(paths) > 0: + dump(f, paths) + +print("****") +print("UNIQUES") +for path in sorted(uniques.keys()): + print(path) diff --git a/hack/lib/branchcuts/test-common.sh b/hack/lib/branchcuts/test-common.sh new file mode 100644 index 00000000000..3790fae8776 --- /dev/null +++ b/hack/lib/branchcuts/test-common.sh @@ -0,0 +1,159 @@ +readonly tty_green='\e[1;32m' +readonly tty_red='\e[1;31m' +readonly tty_blue='\e[1;34m' +readonly tty_nocol='\e[m' + +readonly src_test_files_relative_dir="data" +readonly want_test_files_relative_dir="want" +readonly test_config_relative_path="test_config" + +function branchcuts::read_test_config() { + local -n _cfg=$1 + local _test_config_path="${2}" + for key_value in $(cat "${_test_config_path}") + do + if [[ "${key_value}" =~ (.+?)=(.*) ]] + then + _cfg["${BASH_REMATCH[1]}"]="${BASH_REMATCH[2]}" + fi + done +} +readonly -f branchcuts::read_test_config + +function branchcuts::cleanup() { + local _testRootDir="${1}" + if test -d "${_testRootDir}" + then + os::log:info "deleting %s\n" "${_testRootDir}" + rm --recursive --force "${_testRootDir}" + fi +} +readonly -f branchcuts::cleanup + +function branchcuts::setup_ocp_lifecycle() { + local _ocp_lifecycle_path="${1}" + local _days_till_next_code_free="${2}" + local _current_ocp_release="${3}" + local _future_ocp_release="${4}" + local _next_cf_date=$(date -d "+${_days_till_next_code_free} days" "+%Y-%m-%dT04:00:00Z") + + > "${_ocp_lifecycle_path}" cat << EOF +ocp: + "${_current_ocp_release}": + - event: code-freeze + when: "2022-02-19T02:00:00Z" + "${_future_ocp_release}": + - event: code-freeze + when: "${_next_cf_date}" +EOF +} +readonly -f branchcuts::setup_ocp_lifecycle + +function branchcuts::setup_dirs() { + local _release_repo_absolute_dir="${1}" + local _src_test_files_dir="${2}" + + mkdir --parents "${_release_repo_absolute_dir}" + cp "${_src_test_files_dir}"/* "${_release_repo_absolute_dir}/" +} +readonly -f branchcuts::setup_dirs + +function branchcuts::build_manager() { + local _manager_src_path="${1}" + pushd . + cd "${_manager_src_path}" + os::log:info "building manager ${_manager_src_path}" + go build ./... + popd +} +readonly -f branchcuts::build_manager + +function branchcuts::check_result() { + local _result_files_dir="${1}" + local _want_files_dir="${2}" + local _exit_status=0 + for want_file in $(ls "${_want_files_dir}") + do + local want_file_path="${_want_files_dir}/${want_file}" + local want_file_filename=$(basename "${want_file_path}") + local want_file_in_result_dir="${_result_files_dir}/${want_file_filename}" + if test ! -f "${want_file_in_result_dir}" + then + os::log:info "${tty_red}${want_file_in_result_dir} expected but not found${tty_nocol}" + _exit_status=1 + continue + fi + + os::integration::compare "${want_file_path}" "${want_file_in_result_dir}" + if test $? -ne 0 + then + _exit_status=1 + fi + done + + return $_exit_status +} +readonly -f branchcuts::check_result + +function branchcuts::run_tests_template() { + local run_cfg_manager_fn="${1}" + local _diffs="" + local _exit_status=0 + + os::cmd::expect_success "branchcuts::build_manager '${config_manager_src_path}'" + + for test_case in $(ls "${test_cases_dir}") + do + os::log:info "***************\n" + os::log:info "${tty_blue}Test case: '%s' - Starts${tty_nocol}\n\n" "${test_case}" + local _src_test_files_dir="${test_cases_dir}/${test_case}/${src_test_files_relative_dir}" + local _want_test_files_dir="${test_cases_dir}/${test_case}/${want_test_files_relative_dir}" + + # Read test config from file and set some vars + local _test_config_file_path="${test_cases_dir}/${test_case}/${test_config_relative_path}" + declare -A _test_config + branchcuts::read_test_config _test_config "${_test_config_file_path}" + local _current_ocp_release="${_test_config['current_ocp_release']}" + local _future_ocp_release="${_test_config['future_ocp_release']}" + + # Set test dirs + local _test_root_dir=$(mktemp --tmpdir="${BASETMPDIR}" --directory --suffix "_${test_case}") + local _release_repo_absolute_dir="${_test_root_dir}/${release_repo_relative_dir}" + local _ocp_lifecycle_path="${_release_repo_absolute_dir}/lifecycle.yaml" + + branchcuts::setup_dirs "${_release_repo_absolute_dir}" "${_src_test_files_dir}" + branchcuts::setup_ocp_lifecycle "${_ocp_lifecycle_path}" \ + "${days_till_next_code_free}" "${_current_ocp_release}" "${_future_ocp_release}" + + # Function 'run_config_manager' should be defined into the caller script + if test $(type -t "${run_cfg_manager_fn}") == "function" + then + # "${run_cfg_manager_fn}" "${_test_root_dir}" "${_ocp_lifecycle_path}" _test_config + os::cmd::expect_success "${run_cfg_manager_fn} \"${_test_root_dir}\" \"${_ocp_lifecycle_path}\" _test_config" + else + os::log:error "function '${run_cfg_manager_fn}' is not defined" + exit 1 + fi + + _diffs=$(branchcuts::check_result "${_release_repo_absolute_dir}" "${_want_test_files_dir}") + if test $? -ne 0 + then + os::log:info "${_diffs}\n" + _exit_status=1 + os::log:error "\n${tty_blue}Test case: '%s' - ${tty_red}FAILURE!${tty_nocol}\n" "${test_case}" + else + os::log:info "\n${tty_blue}Test case: '%s' - ${tty_green}SUCCESS${tty_nocol}\n" "${test_case}" + fi + os::log:info "***************\n\n" + done + + # Clean tests folder only if everything went well. + # Otherwise it reserve some time to inspect the failure. + if test $_exit_status -eq 0 + then + branchcuts::cleanup "${_test_root_dir}" + fi + + return $_exit_status +} +readonly -f branchcuts::run_tests_template \ No newline at end of file diff --git a/pkg/api/ocplifecycle/codefreeze.go b/pkg/api/ocplifecycle/codefreeze.go new file mode 100644 index 00000000000..17ea41487d7 --- /dev/null +++ b/pkg/api/ocplifecycle/codefreeze.go @@ -0,0 +1,71 @@ +package ocplifecycle + +import ( + "time" + + "github.com/sirupsen/logrus" + + "k8s.io/apimachinery/pkg/util/sets" +) + +const ( + ocpProductName = "ocp" +) + +type OCPCodeFreezeLifecycle struct { + c *Config + timelineOpts TimelineOptions + timeline Timeline + when time.Time + IsCloseToCodeFreeze func(codeFreezeDate, now time.Time, howManyDaysBefore int) bool +} + +func NewOCPCodeFreezeLifecycle(c *Config, when time.Time) *OCPCodeFreezeLifecycle { + tOpts := TimelineOptions{ + OnlyEvents: sets.NewString([]string{ + string(LifecycleEventCodeFreeze), + }...), + } + t := c.GetTimeline(ocpProductName, tOpts) + return &OCPCodeFreezeLifecycle{ + c: c, + timelineOpts: tOpts, + timeline: t, + when: when, + IsCloseToCodeFreeze: isCloseToCodeFreeze, + } +} + +func (ol *OCPCodeFreezeLifecycle) IsBeforeCodeFreeze(howManyDaysBefore int) bool { + _, after := ol.timeline.DeterminePlaceInTime(ol.when) + if after.LifecyclePhase.Event != LifecycleEventCodeFreeze { + return false + } + return ol.IsCloseToCodeFreeze(after.LifecyclePhase.When.Time, ol.when, howManyDaysBefore) +} + +func (ol *OCPCodeFreezeLifecycle) GetOCPVersions() (cur, future *MajorMinor, e error) { + before, after := ol.timeline.DeterminePlaceInTime(ol.when) + logrus.Debugf("parsing current OCP version %s", before.ProductVersion) + currentVersion, err := ParseMajorMinor(before.ProductVersion) + if err != nil { + return nil, nil, err + } + + logrus.Debugf("parsing future OCP version %s", after.ProductVersion) + futureVersion, err := ParseMajorMinor(after.ProductVersion) + if err != nil { + return nil, nil, err + } + return currentVersion, futureVersion, nil +} + +func isCloseToCodeFreeze(codeFreezeDate, now time.Time, howManyDaysBefore int) bool { + distance := codeFreezeDate.Sub(now) + daysBefore := int(distance.Hours() / 24) + if howManyDaysBefore := int(distance.Hours() / 24); daysBefore == howManyDaysBefore { + return true + } + logrus.Debugf("days before code freeze %d, skipping", howManyDaysBefore) + return false +} diff --git a/pkg/branchcuts/bumper/bumper.go b/pkg/branchcuts/bumper/bumper.go new file mode 100644 index 00000000000..1252d9d9b48 --- /dev/null +++ b/pkg/branchcuts/bumper/bumper.go @@ -0,0 +1,106 @@ +package bumper + +import ( + "errors" + "fmt" + "os" + "path" + + "github.com/sirupsen/logrus" + + utilerrors "k8s.io/apimachinery/pkg/util/errors" +) + +type Bumper[T any] interface { + GetFiles() ([]string, error) + + Unmarshall(file string) (T, error) + + BumpFilename(filename string, obj T) (string, error) + + BumpContent(obj T) (T, error) + + Marshall(obj T, bumpedFilename, dir string) error +} + +type BumpingOptions struct { + OutDir string +} + +// Bump bumps files using the Bumpers b according to the BumpingOptions. +func Bump[T any](b Bumper[T], o *BumpingOptions) (bool, error) { + var errs []error + + files, err := b.GetFiles() + logrus.Debugf("files: %+v", files) + if err != nil { + return false, err + } + + for _, f := range files { + if err := BumpObject(b, f, o.OutDir); err != nil { + errs = append(errs, err) + } + } + + if err = utilerrors.NewAggregate(errs); err != nil { + return false, err + } + + return true, nil +} + +func BumpObject[T any](b Bumper[T], file, outDir string) (retErr error) { + logrus.Infof("bumping config %s", file) + + srcFileFullPath := file + + obj, err := b.Unmarshall(srcFileFullPath) + if err != nil { + logrus.WithError(err).Errorf("failed to unmarshall file %s", srcFileFullPath) + return fmt.Errorf("unmarshall file %s: %w", file, err) + } + + filename := path.Base(file) + + logrus.Infof("bumping filename %s", filename) + bumpedFilename, err := b.BumpFilename(filename, obj) + if err != nil { + logrus.WithError(err).Errorf("error bumping file %s", bumpedFilename) + return fmt.Errorf("bump filename: %w", err) + } + logrus.Infof("bumped filename %s", bumpedFilename) + + outDir = getOutDir(file, outDir) + logrus.Debugf("out dir: %s", outDir) + dstFileFullPath := path.Join(outDir, bumpedFilename) + + if _, err := os.Stat(dstFileFullPath); err == nil { + logrus.WithError(err).Warnf("file %s already exists, skipping", dstFileFullPath) + return NewFileExistsErr(dstFileFullPath) + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("file exists: %w", err) + } + + logrus.Infof("bumping obj") + bumpedObj, err := b.BumpContent(obj) + if err != nil { + logrus.WithError(err).Error("error bumping obj") + return fmt.Errorf("bump object: %w", err) + } + + logrus.Infof("marshalling obj %s to %s", bumpedFilename, outDir) + if err := b.Marshall(bumpedObj, bumpedFilename, outDir); err != nil { + logrus.WithError(err).Error("error marshalling obj") + return fmt.Errorf("marshall obj: %w", err) + } + + return nil +} + +func getOutDir(file string, dir string) string { + if dir != "" { + return dir + } + return path.Dir(file) +} diff --git a/pkg/branchcuts/bumper/errors.go b/pkg/branchcuts/bumper/errors.go new file mode 100644 index 00000000000..06799fab4aa --- /dev/null +++ b/pkg/branchcuts/bumper/errors.go @@ -0,0 +1,73 @@ +package bumper + +import "fmt" + +type UnmarshallError struct { + file string + err error +} + +func NewUnmarshallError(f string, err error) error { + return &UnmarshallError{ + f, + err, + } +} + +func (e *UnmarshallError) Error() string { + return e.file + ": " + e.err.Error() +} + +func (e *UnmarshallError) Is(err error) bool { + if e == nil || err == nil { + return e == err + } + _, ok := err.(*UnmarshallError) + return ok +} + +type ConfigSaveError struct { + file string + err error +} + +func NewConfigSaveError(f string, err error) error { + return &ConfigSaveError{ + f, + err, + } +} + +func (e *ConfigSaveError) Error() string { + return e.file + ": " + e.err.Error() +} + +func (e *ConfigSaveError) Is(err error) bool { + if e == nil || err == nil { + return e == err + } + _, ok := err.(*ConfigSaveError) + return ok +} + +type FileExistsErr struct { + msg string +} + +func NewFileExistsErr(f string) *FileExistsErr { + return &FileExistsErr{ + fmt.Sprintf("file %s already exists", f), + } +} + +func (fee *FileExistsErr) Error() string { + return fee.msg +} + +func (fee *FileExistsErr) Is(e error) bool { + if e == nil || fee == nil { + return e == fee + } + _, ok := e.(*FileExistsErr) + return ok +} diff --git a/pkg/branchcuts/bumper/gen-release-jobs-bumper.go b/pkg/branchcuts/bumper/gen-release-jobs-bumper.go new file mode 100644 index 00000000000..ae354b18102 --- /dev/null +++ b/pkg/branchcuts/bumper/gen-release-jobs-bumper.go @@ -0,0 +1,241 @@ +package bumper + +import ( + "fmt" + "io/fs" + "path" + "path/filepath" + "regexp" + "strconv" + "strings" + + "github.com/sirupsen/logrus" + + cioperatorapi "github.com/openshift/ci-tools/pkg/api" + "github.com/openshift/ci-tools/pkg/api/ocplifecycle" + cioperatorcfg "github.com/openshift/ci-tools/pkg/config" +) + +const ( + releaseJobsRegexPatternFormat = `openshift-release-master__(okd|ci|nightly)-%s.*\.yaml` + ocpReleaseEnvVarName = "OCP_VERSION" +) + +type GeneratedReleaseGatingJobsBumper struct { + mm *ocplifecycle.MajorMinor + getFilesRegexp *regexp.Regexp + jobsDir string + newIntervalValue int +} + +var _ Bumper[*cioperatorcfg.DataWithInfo] = &GeneratedReleaseGatingJobsBumper{} + +func NewGeneratedReleaseGatingJobsBumper(ocpVer, jobsDir string, newIntervalValue int) *GeneratedReleaseGatingJobsBumper { + mm, err := ocplifecycle.ParseMajorMinor(ocpVer) + if err != nil { + panic(err) + } + mmRegexp := fmt.Sprintf("%d\\.%d", mm.Major, mm.Minor) + getFilesRegexp := regexp.MustCompile(fmt.Sprintf(releaseJobsRegexPatternFormat, mmRegexp)) + return &GeneratedReleaseGatingJobsBumper{ + mm, + getFilesRegexp, + jobsDir, + newIntervalValue, + } +} + +func (b *GeneratedReleaseGatingJobsBumper) GetFiles() ([]string, error) { + files := make([]string, 0) + err := filepath.Walk(b.jobsDir, func(path string, info fs.FileInfo, err error) error { + if b.getFilesRegexp.Match([]byte(path)) { + files = append(files, path) + } + return nil + }) + return files, err +} + +func (b *GeneratedReleaseGatingJobsBumper) Unmarshall(file string) (*cioperatorcfg.DataWithInfo, error) { + cfgDataByFilename, err := cioperatorcfg.LoadDataByFilename(file) + if err != nil { + return nil, err + } + + filename := path.Base(file) + dataWithInfo, ok := cfgDataByFilename[filename] + if !ok { + logrus.WithError(err).Errorf("failed to get config %s", file) + return nil, NewUnmarshallError(file, err) + } + return &dataWithInfo, nil +} + +func (b *GeneratedReleaseGatingJobsBumper) BumpFilename( + filename string, + dataWithInfo *cioperatorcfg.DataWithInfo) (string, error) { + newVariant, err := ReplaceWithNextVersion(dataWithInfo.Info.Metadata.Variant, b.mm.Major) + if err != nil { + return "", err + } + dataWithInfo.Info.Metadata.Variant = newVariant + return dataWithInfo.Info.Metadata.Basename(), nil +} + +/* + Candidate bumping fields: + .base_images.*.name + .releases.*.{release,candidate}.version + .releases.*.prerelease.version_bounds.{lower,upper} + .tests[].steps.test[].env[].default +*/ +func (b *GeneratedReleaseGatingJobsBumper) BumpContent(dataWithInfo *cioperatorcfg.DataWithInfo) (*cioperatorcfg.DataWithInfo, error) { + major := b.mm.Major + config := &dataWithInfo.Configuration + if err := bumpBaseImages(config, major); err != nil { + return nil, err + } + + if err := bumpReleases(config, major); err != nil { + return nil, err + } + + if err := bumpTests(config, major); err != nil { + return nil, err + } + + if err := ReplaceWithNextVersionInPlace(&config.Metadata.Variant, major); err != nil { + return nil, err + } + + if config.Tests != nil { + for i := 0; i < len(config.Tests); i++ { + if config.Tests[i].Interval != nil { + *config.Tests[i].Interval = strconv.Itoa(b.newIntervalValue) + "h" + } + } + } + + return dataWithInfo, nil +} + +func bumpBaseImages(config *cioperatorapi.ReleaseBuildConfiguration, major int) error { + if config.BaseImages == nil { + return nil + } + + bumpedImages := make(map[string]cioperatorapi.ImageStreamTagReference) + for k := range config.BaseImages { + image := config.BaseImages[k] + if err := ReplaceWithNextVersionInPlace(&image.Name, major); err != nil { + return err + } + bumpedImages[k] = image + } + config.BaseImages = bumpedImages + return nil +} + +func bumpReleases(config *cioperatorapi.ReleaseBuildConfiguration, major int) error { + if config.Releases == nil { + return nil + } + + bumpedReleases := make(map[string]cioperatorapi.UnresolvedRelease) + for k := range config.Releases { + release := config.Releases[k] + if release.Release != nil { + if err := ReplaceWithNextVersionInPlace(&release.Release.Version, major); err != nil { + return err + } + } + if release.Candidate != nil { + if err := ReplaceWithNextVersionInPlace(&release.Candidate.Version, major); err != nil { + return err + } + } + if release.Prerelease != nil { + if err := ReplaceWithNextVersionInPlace(&release.Prerelease.VersionBounds.Upper, major); err != nil { + return err + } + if err := ReplaceWithNextVersionInPlace(&release.Prerelease.VersionBounds.Lower, major); err != nil { + return err + } + } + bumpedReleases[k] = release + } + config.Releases = bumpedReleases + return nil +} + +func bumpTests(config *cioperatorapi.ReleaseBuildConfiguration, major int) error { + if config.Tests == nil { + return nil + } + + for i := 0; i < len(config.Tests); i++ { + test := config.Tests[i] + + if test.MultiStageTestConfiguration == nil { + continue + } + + if err := bumpTestSteps(test.MultiStageTestConfiguration.Pre, major); err != nil { + return err + } + if err := bumpTestSteps(test.MultiStageTestConfiguration.Test, major); err != nil { + return err + } + if err := bumpTestSteps(test.MultiStageTestConfiguration.Post, major); err != nil { + return err + } + + if err := ReplaceWithNextVersionInPlace(test.MultiStageTestConfiguration.Workflow, major); err != nil { + return err + } + + config.Tests[i] = test + } + + return nil +} + +func bumpTestSteps(tests []cioperatorapi.TestStep, major int) error { + if tests == nil { + return nil + } + for i := 0; i < len(tests); i++ { + multistageTest := tests[i] + if err := bumpTestStepEnvVars(multistageTest, major); err != nil { + return err + } + tests[i] = multistageTest + } + return nil +} + +func bumpTestStepEnvVars(multistageTest cioperatorapi.TestStep, major int) error { + if multistageTest.LiteralTestStep == nil || multistageTest.LiteralTestStep.Environment == nil { + return nil + } + for i := 0; i < len(multistageTest.Environment); i++ { + env := multistageTest.Environment[i] + if env.Name == ocpReleaseEnvVarName { + if err := ReplaceWithNextVersionInPlace(env.Default, major); err != nil { + return err + } + } + multistageTest.Environment[i] = env + } + return nil +} + +func (b *GeneratedReleaseGatingJobsBumper) Marshall(dataWithInfo *cioperatorcfg.DataWithInfo, bumpedFilename, dir string) error { + absolutePath := path.Join(dir, bumpedFilename) + dirNoOrgRepo := strings.TrimSuffix(absolutePath, dataWithInfo.Info.Metadata.RelativePath()) + if err := dataWithInfo.CommitTo(dirNoOrgRepo); err != nil { + logrus.WithError(err).Errorf("error saving config %s", dirNoOrgRepo) + return NewConfigSaveError(dirNoOrgRepo, err) + } + return nil +} diff --git a/pkg/branchcuts/bumper/gen-release-jobs-bumper_test.go b/pkg/branchcuts/bumper/gen-release-jobs-bumper_test.go new file mode 100644 index 00000000000..9c33e897d44 --- /dev/null +++ b/pkg/branchcuts/bumper/gen-release-jobs-bumper_test.go @@ -0,0 +1,174 @@ +package bumper + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + + cioperatorapi "github.com/openshift/ci-tools/pkg/api" + cioperatorcfg "github.com/openshift/ci-tools/pkg/config" +) + +func TestBumpFilename(t *testing.T) { + tests := []struct { + id string + ocpRelease string + meta cioperatorapi.Metadata + wantMeta cioperatorapi.Metadata + }{ + { + id: "Bump filename properly", + ocpRelease: "4.11", + meta: cioperatorapi.Metadata{ + Variant: "4.10", + }, + wantMeta: cioperatorapi.Metadata{ + Variant: "4.11", + }, + }, + } + for _, test := range tests { + t.Run(test.id, func(t *testing.T) { + // TODO + }) + } +} + +func TestBumpObject(t *testing.T) { + tests := []struct { + id string + ocpRelease string + interval int + config cioperatorapi.ReleaseBuildConfiguration + wantConfig cioperatorapi.ReleaseBuildConfiguration + }{ + { + id: "Object bumped properly", + ocpRelease: "4.11", + interval: 168, + config: cioperatorapi.ReleaseBuildConfiguration{ + Metadata: cioperatorapi.Metadata{ + Variant: "4.10", + }, + InputConfiguration: cioperatorapi.InputConfiguration{ + BaseImages: map[string]cioperatorapi.ImageStreamTagReference{ + "image-1": { + Name: "image_4.10", + }, + }, + Releases: map[string]cioperatorapi.UnresolvedRelease{ + "release": { + Release: &cioperatorapi.Release{ + Version: "4.10", + }, + }, + "candidate": { + Release: &cioperatorapi.Release{ + Version: "4.10", + }, + }, + "prerelease": { + Prerelease: &cioperatorapi.Prerelease{ + VersionBounds: cioperatorapi.VersionBounds{ + Lower: "4.9", + Upper: "4.11", + }, + }, + }, + }, + }, + Tests: []cioperatorapi.TestStepConfiguration{ + { + Interval: strRef("24h"), + MultiStageTestConfiguration: &cioperatorapi.MultiStageTestConfiguration{ + Workflow: strRef("w-4.10-4.11"), + Test: []cioperatorapi.TestStep{ + { + LiteralTestStep: &cioperatorapi.LiteralTestStep{ + Environment: []cioperatorapi.StepParameter{ + { + Name: "OCP_VERSION", + Default: strRef("4.10"), + }, + }, + }, + }, + }, + }, + }, + }, + }, + wantConfig: cioperatorapi.ReleaseBuildConfiguration{ + Metadata: cioperatorapi.Metadata{ + Variant: "4.11", + }, + InputConfiguration: cioperatorapi.InputConfiguration{ + BaseImages: map[string]cioperatorapi.ImageStreamTagReference{ + "image-1": { + Name: "image_4.11", + }, + }, + Releases: map[string]cioperatorapi.UnresolvedRelease{ + "release": { + Release: &cioperatorapi.Release{ + Version: "4.11", + }, + }, + "candidate": { + Release: &cioperatorapi.Release{ + Version: "4.11", + }, + }, + "prerelease": { + Prerelease: &cioperatorapi.Prerelease{ + VersionBounds: cioperatorapi.VersionBounds{ + Lower: "4.10", + Upper: "4.12", + }, + }, + }, + }, + }, + Tests: []cioperatorapi.TestStepConfiguration{ + { + Interval: strRef("168h"), + MultiStageTestConfiguration: &cioperatorapi.MultiStageTestConfiguration{ + Workflow: strRef("w-4.11-4.12"), + Test: []cioperatorapi.TestStep{ + { + LiteralTestStep: &cioperatorapi.LiteralTestStep{ + Environment: []cioperatorapi.StepParameter{ + { + Name: "OCP_VERSION", + Default: strRef("4.11"), + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + for _, test := range tests { + t.Run(test.id, func(t *testing.T) { + b := NewGeneratedReleaseGatingJobsBumper(test.ocpRelease, "", test.interval) + dataWithInfo := cioperatorcfg.DataWithInfo{ + Configuration: test.config, + } + result, err := b.BumpContent(&dataWithInfo) + if err != nil { + t.Errorf("Unexpected error: %s", err.Error()) + } + if diff := cmp.Diff(&test.wantConfig, &result.Configuration); diff != "" { + t.Errorf("Configurations are different: %s", diff) + } + }) + } +} + +func strRef(i string) *string { + return &i +} diff --git a/pkg/branchcuts/bumper/replace.go b/pkg/branchcuts/bumper/replace.go new file mode 100644 index 00000000000..244507a561f --- /dev/null +++ b/pkg/branchcuts/bumper/replace.go @@ -0,0 +1,78 @@ +package bumper + +import ( + "fmt" + "regexp" + "sort" + "strconv" + "strings" +) + +func ReplaceWithNextVersionInPlace(line *string, major int) error { + newLine, err := ReplaceWithNextVersion(*line, major) + if err != nil { + return err + } + *line = newLine + return nil +} + +// Find every {major}.{minor} reference into 'line' and the replace it +// with {major}.{minor+1} +func ReplaceWithNextVersion(line string, major int) (string, error) { + p := fmt.Sprintf(`%d\.(?P\d+)`, major) + r := regexp.MustCompile(p) + m := r.FindAllStringSubmatch(line, -1) + if m == nil { + return line, nil + } + + minors, err := uniqueSortedMinors(m) + if err != nil { + return line, err + } + + for _, minor := range minors { + curVersion := fmt.Sprintf("%d.%d", major, minor) + nextVersion := fmt.Sprintf("%d.%d", major, minor+1) + line = strings.ReplaceAll(line, curVersion, nextVersion) + } + + return line, nil +} + +// The function extracts all the minors from matches, removes duplicate and finally it returs them in +// decreasing order. +// +// Matches is an array of regex matches that could be obtained by the following example: +// string: ocp_4.5-4.6-4.5 +// pattern: 4\.\d+ +// matches: [3]string{} +// matches[0]: []string{ "4.5", "5" } +// matches[1]: []string{ "4.6", "6" } +// matches[2]: []string{ "4.5", "5" } +// +// Given the previous input, the function returns []int{6, 5} +func uniqueSortedMinors(matches [][]string) ([]int, error) { + minors := make(map[int]bool) + for _, m := range matches { + for i := 1; i < len(m); i++ { + minor, err := strconv.Atoi(m[i]) + if err != nil { + return []int{}, err + } + minors[minor] = true + } + } + + keys := make([]int, 0, len(minors)) + for k := range minors { + keys = append(keys, k) + } + + sort.Slice(keys, func(i, j int) bool { + return !(keys[i] < keys[j]) + }) + + return keys, nil +} diff --git a/pkg/branchcuts/bumper/replace_test.go b/pkg/branchcuts/bumper/replace_test.go new file mode 100644 index 00000000000..bfc504953d8 --- /dev/null +++ b/pkg/branchcuts/bumper/replace_test.go @@ -0,0 +1,70 @@ +package bumper_test + +import ( + "testing" + + "github.com/openshift/ci-tools/pkg/branchcuts/bumper" +) + +func TestReplaceWithNextVersion(t *testing.T) { + tests := []struct { + name string + line string + major int + expected string + }{ + { + "Bump to the next version properly", + "product_3.2", + 3, + "product_3.3", + }, + { + "Bump skipped due to major mismatch", + "product_3.2", + 2, + "product_3.2", + }, + { + "Unable to bump when leading zeroes", + "product_3.002", + 3, + "product_3.002", + }, + { + "Multiple bumping", + "product_3.2 product_3.9", + 3, + "product_3.3 product_3.10", + }, + { + "Multiple bumping 2", + "openshift-upgrade-ovirt-release-4.5-4.6", + 4, + "openshift-upgrade-ovirt-release-4.6-4.7", + }, + { + "Multiple bumping with a major mismatch", + "product_3.2 product_3.9 product_4.1", + 3, + "product_3.3 product_3.10 product_4.1", + }, + { + "Unexpected dot", + "product_3..2", + 3, + "product_3..2", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + l, err := bumper.ReplaceWithNextVersion(test.line, test.major) + if err != nil { + t.Error(err) + } else if l != test.expected { + t.Errorf("Expected %s but got %s", test.expected, l) + } + }) + } +} diff --git a/pkg/branchcuts/bumper/repo-bumper.go b/pkg/branchcuts/bumper/repo-bumper.go new file mode 100644 index 00000000000..de6e3f3a6a3 --- /dev/null +++ b/pkg/branchcuts/bumper/repo-bumper.go @@ -0,0 +1,148 @@ +package bumper + +import ( + "bytes" + "fmt" + "io/fs" + "io/ioutil" + "os" + "path" + "strings" + + "gopkg.in/ini.v1" + + "github.com/openshift/ci-tools/pkg/api/ocplifecycle" +) + +type ( + RepoBumper struct { + GlobPattern string + FilesDir string + OCPRelease ocplifecycle.MajorMinor + } + + RepoBumperOptions struct { + GlobPattern string + FilesDir string + CurOCPRelease string + } + + rhelRepo struct { + BaseUrl string `ini:"baseurl,omitempty"` + Enabled int `ini:"baseurl,omitempty"` + FailoverMethod bool `ini:"failovermethod,omitempty"` + GPGCheck int `ini:"gpgcheck,omitempty"` + GPGKey string `ini:"gpgkey,omitempty"` + Name string `ini:"name,omitempty"` + PasswordFile string `ini:"password_file,omitempty"` + SkipIfUnavailable bool `ini:"skip_if_unavailable,omitempty"` + SSLClientCert string `ini:"sslclientcert,omitempty"` + SSLClientKey string `ini:"sslclientkey,omitempty"` + SSLVerify bool `ini:"sslverify,omitempty"` + UsernameFile string `ini:"username_file,omitempty"` + } +) + +var ( + majorMinorSeparators = []string{".", "-"} + _ Bumper[*ini.File] = &RepoBumper{} +) + +func NewRepoBumper(o *RepoBumperOptions) *RepoBumper { + mm, err := ocplifecycle.ParseMajorMinor(o.CurOCPRelease) + if err != nil { + panic(fmt.Sprintf("Error parsing %s", o.CurOCPRelease)) + } + return &RepoBumper{ + GlobPattern: o.GlobPattern, + FilesDir: o.FilesDir, + OCPRelease: *mm, + } +} + +func (b *RepoBumper) GetFiles() ([]string, error) { + dirFs := os.DirFS(b.FilesDir) + matches, err := fs.Glob(dirFs, b.GlobPattern) + if err != nil { + return nil, err + } + files := make([]string, 0, len(matches)) + for _, f := range matches { + fileFullPath := path.Join(b.FilesDir, f) + files = append(files, fileFullPath) + } + return files, nil +} + +func (b *RepoBumper) Unmarshall(file string) (*ini.File, error) { + return ini.Load(file) +} + +func (b *RepoBumper) BumpFilename(filename string, _ *ini.File) (string, error) { + curRelease := fmt.Sprintf("%d.%d", b.OCPRelease.Major, b.OCPRelease.Minor) + futureRelease := fmt.Sprintf("%d.%d", b.OCPRelease.Major, b.OCPRelease.Minor+1) + return strings.ReplaceAll(filename, curRelease, futureRelease), nil +} + +func (b *RepoBumper) BumpContent(file *ini.File) (*ini.File, error) { + for _, section := range file.Sections() { + repo := rhelRepo{} + if err := section.MapTo(&repo); err != nil { + return nil, err + } + + for _, s := range majorMinorSeparators { + curRelease := fmt.Sprintf("%d%s%d", b.OCPRelease.Major, s, b.OCPRelease.Minor) + futureRelease := fmt.Sprintf("%d%s%d", b.OCPRelease.Major, s, b.OCPRelease.Minor+1) + repo.BaseUrl = strings.ReplaceAll(repo.BaseUrl, curRelease, futureRelease) + } + + if err := section.ReflectFrom(&repo); err != nil { + return nil, err + } + } + return file, nil +} + +func (b *RepoBumper) Marshall(file *ini.File, bumpedFilename, dir string) error { + filePath := path.Join(dir, bumpedFilename) + return saveIniFile(filePath, file) +} + +func saveIniFile(path string, f *ini.File) error { + ini.PrettySection = true + ini.PrettyFormat = false + ini.PrettyEqual = true + + // What follow should have be avoided by using f.SaveTo(path) directly, + // but unfortunately it appends a double '\n' at the end of the file + // that makes it different from the original one: we should only bump the fields of + // interest without doing anything else. + // Consider opening a PR that fixes this issue, even if I'm not sure this can be + // considered an issue. + buf := bytes.NewBuffer(nil) + if _, err := f.WriteTo(buf); err != nil { + return err + } + bs := buf.Bytes() + if hasSuffix(bs, []byte(ini.LineBreak)) { + bs = bs[0 : len(bs)-len(ini.LineBreak)] + } + if err := ioutil.WriteFile(path, bs, 0666); err != nil { + return err + } + + return nil +} + +func hasSuffix(data, suffix []byte) bool { + if len(suffix) > len(data) { + return false + } + dataLen := len(data) + suffixLen := len(suffix) + var i int + for i = 0; i < suffixLen && data[dataLen-i-1] == suffix[suffixLen-i-1]; i++ { + } + return i == suffixLen +} diff --git a/pkg/branchcuts/bumper/repo-bumper_test.go b/pkg/branchcuts/bumper/repo-bumper_test.go new file mode 100644 index 00000000000..210f193a364 --- /dev/null +++ b/pkg/branchcuts/bumper/repo-bumper_test.go @@ -0,0 +1,77 @@ +package bumper_test + +import ( + "strings" + "testing" + + "gopkg.in/ini.v1" + + "github.com/openshift/ci-tools/pkg/branchcuts/bumper" +) + +func TestBumpRepo(t *testing.T) { + tests := []struct { + id string + content string + wantContent string + curOCPRelease string + }{ + { + "Bump properly", + `[rhel-repo-1] +name = test +baseurl = https://fake-repo.com/ocp/4.10/test +`, + `[rhel-repo-1] +name = test +baseurl = https://fake-repo.com/ocp/4.11/test + +`, + "4.10", + }, + { + "Bump dash", + `[rhel-repo-1] +name = test +baseurl = https://fake-repo.com/ocp/4-10/test +`, + `[rhel-repo-1] +name = test +baseurl = https://fake-repo.com/ocp/4-11/test + +`, + "4.10", + }, + } + + ini.PrettySection = true + ini.PrettyFormat = false + ini.PrettyEqual = true + + for _, test := range tests { + t.Run(test.id, func(t *testing.T) { + t.Parallel() + b := bumper.NewRepoBumper(&bumper.RepoBumperOptions{ + CurOCPRelease: test.curOCPRelease, + }) + file, err := ini.Load([]byte(test.content)) + if err != nil { + t.Error(err) + } + + result, err := b.BumpContent(file) + if err != nil { + t.Error(err) + } else { + buf := strings.Builder{} + if _, err := result.WriteTo(&buf); err != nil { + t.Error(err) + } + resultContent := buf.String() + if test.wantContent != resultContent { + t.Errorf("Expected '%s' but got '%s'\n", test.wantContent, resultContent) + } + } + }) + } +} diff --git a/pkg/branchcuts/config-manager/manager.go b/pkg/branchcuts/config-manager/manager.go new file mode 100644 index 00000000000..aa993b717ee --- /dev/null +++ b/pkg/branchcuts/config-manager/manager.go @@ -0,0 +1,9 @@ +package configmanager + +type ReconcileStatus uint8 + +const ( + ReconcileOk ReconcileStatus = iota + ReconcileSkipped + ReconcileErr = 255 +) diff --git a/test/integration/br-cuts-generated-release-jobs.sh b/test/integration/br-cuts-generated-release-jobs.sh new file mode 100755 index 00000000000..51e27171f10 --- /dev/null +++ b/test/integration/br-cuts-generated-release-jobs.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +source "$(dirname "${BASH_SOURCE}")/../../hack/lib/init.sh" + +readonly release_repo_relative_dir="ci-operator/config/openshift/release" +readonly config_manager_src_path="${OS_ROOT}/cmd/branchingconfigmanagers/generated-release-gating-jobs" +readonly config_manager_bin_path="${config_manager_src_path}/generated-release-gating-jobs" + +readonly test_cases_dir="${OS_ROOT}/test/integration/branchcuts/generated-release-gating-jobs" +readonly days_till_next_code_free=14 + +function cleanup() { + os::test::junit::reconcile_output + os::cleanup::processes +} +trap "cleanup" EXIT + +function branchcuts::generated_release_jobs::run_config_manager() { + local _config_manager_bin_path="${config_manager_bin_path}" + local _release_repo_path="${1}" + local _ocp_lifecycle_path="${2}" + local -n _cfg=$3 + local _interval="${_cfg['interval']}" + + printf "%s\n%s %s\n%s %s\n%s %s\n\n" \ + $(basename "${_config_manager_bin_path}") \ + --release-repo-path "${_release_repo_path}" \ + --lifecycle-config "${_ocp_lifecycle_path}" \ + --interval "${_interval}" + + "${_config_manager_bin_path}" \ + --release-repo-path "${_release_repo_path}" \ + --lifecycle-config "${_ocp_lifecycle_path}" \ + --interval "${_interval}" +} +readonly -f branchcuts::generated_release_jobs::run_config_manager + +function branchcuts::generated_release_jobs::run_tests() { + branchcuts::run_tests_template branchcuts::generated_release_jobs::run_config_manager +} +readonly -f branchcuts::generated_release_jobs::run_tests + + +os::test::junit::declare_suite_start "integration/branchcuts/generated-release-gating-jobs" +os::cmd::expect_success "branchcuts::generated_release_jobs::run_tests" +os::test::junit::declare_suite_end + diff --git a/test/integration/br-cuts-rpm-mirroring-services.sh b/test/integration/br-cuts-rpm-mirroring-services.sh new file mode 100755 index 00000000000..0d31ad74709 --- /dev/null +++ b/test/integration/br-cuts-rpm-mirroring-services.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +source "$(dirname "${BASH_SOURCE}")/../../hack/lib/init.sh" + +readonly release_repo_relative_dir="core-services/release-controller/_repos" +readonly config_manager_src_path="${OS_ROOT}/cmd/branchingconfigmanagers/rpm-deps-mirroring-services" +readonly config_manager_bin_path="${config_manager_src_path}/rpm-deps-mirroring-services" + +readonly test_cases_dir="${OS_ROOT}/test/integration/branchcuts/rpm-deps-mirroring-services" +readonly days_till_next_code_free=14 + +function cleanup() { + os::test::junit::reconcile_output + os::cleanup::processes +} +trap "cleanup" EXIT + +function branchcuts::rpm_mirroring::run_config_manager() { + local _config_manager_bin_path="${config_manager_bin_path}" + local _release_repo_path="${1}" + local _ocp_lifecycle_path="${2}" + local -n _cfg=$3 + + printf "%s\n%s %s\n%s %s\n%s %s\n\n" \ + $(basename "${_config_manager_bin_path}") \ + --release-repo-path "${_release_repo_path}" \ + --lifecycle-config "${_ocp_lifecycle_path}" + + "${_config_manager_bin_path}" \ + --release-repo-path "${_release_repo_path}" \ + --lifecycle-config "${_ocp_lifecycle_path}" +} +readonly -f branchcuts::rpm_mirroring::run_config_manager + + +function branchcuts::rpm_mirroring::run_tests() { + branchcuts::run_tests_template branchcuts::rpm_mirroring::run_config_manager +} +readonly -f branchcuts::rpm_mirroring::run_tests + +os::test::junit::declare_suite_start "integration/branchcuts/rpm-deps-mirroring-services" +os::cmd::expect_success "branchcuts::rpm_mirroring::run_tests" +os::test::junit::declare_suite_end diff --git a/test/integration/branchcuts/generated-release-gating-jobs/test1/data/openshift-release-master__ci-4.10.yaml b/test/integration/branchcuts/generated-release-gating-jobs/test1/data/openshift-release-master__ci-4.10.yaml new file mode 100644 index 00000000000..976f6c1c10b --- /dev/null +++ b/test/integration/branchcuts/generated-release-gating-jobs/test1/data/openshift-release-master__ci-4.10.yaml @@ -0,0 +1,37 @@ +releases: + initial: + candidate: + product: ocp + relative: 1 + stream: ci + version: "4.10" + latest: + candidate: + product: ocp + stream: ci + version: "4.10" +resources: + '*': + requests: + cpu: 100m + memory: 200Mi +tests: +- as: e2e-aws + interval: 24h + steps: + cluster_profile: aws-2 + env: + BASE_DOMAIN: aws-2.ci.openshift.org + workflow: openshift-e2e-aws +- as: e2e-aws-techpreview + interval: 48h + steps: + cluster_profile: aws-2 + env: + BASE_DOMAIN: aws-2.ci.openshift.org + workflow: openshift-e2e-aws-techpreview +zz_generated_metadata: + branch: master + org: openshift + repo: release + variant: ci-4.10 diff --git a/test/integration/branchcuts/generated-release-gating-jobs/test1/test_config b/test/integration/branchcuts/generated-release-gating-jobs/test1/test_config new file mode 100644 index 00000000000..6a2938b6499 --- /dev/null +++ b/test/integration/branchcuts/generated-release-gating-jobs/test1/test_config @@ -0,0 +1,3 @@ +current_ocp_release=4.10 +future_ocp_release=4.11 +interval=168 \ No newline at end of file diff --git a/test/integration/branchcuts/generated-release-gating-jobs/test1/want/openshift-release-master__ci-4.11.yaml b/test/integration/branchcuts/generated-release-gating-jobs/test1/want/openshift-release-master__ci-4.11.yaml new file mode 100644 index 00000000000..27bc85264e2 --- /dev/null +++ b/test/integration/branchcuts/generated-release-gating-jobs/test1/want/openshift-release-master__ci-4.11.yaml @@ -0,0 +1,37 @@ +releases: + initial: + candidate: + product: ocp + relative: 1 + stream: ci + version: "4.11" + latest: + candidate: + product: ocp + stream: ci + version: "4.11" +resources: + '*': + requests: + cpu: 100m + memory: 200Mi +tests: +- as: e2e-aws + interval: 168h + steps: + cluster_profile: aws-2 + env: + BASE_DOMAIN: aws-2.ci.openshift.org + workflow: openshift-e2e-aws +- as: e2e-aws-techpreview + interval: 168h + steps: + cluster_profile: aws-2 + env: + BASE_DOMAIN: aws-2.ci.openshift.org + workflow: openshift-e2e-aws-techpreview +zz_generated_metadata: + branch: master + org: openshift + repo: release + variant: ci-4.11 diff --git a/test/integration/branchcuts/generated-release-gating-jobs/test2/data/openshift-release-master__ci-4.5-upgrade-from-stable-4.4.yaml b/test/integration/branchcuts/generated-release-gating-jobs/test2/data/openshift-release-master__ci-4.5-upgrade-from-stable-4.4.yaml new file mode 100644 index 00000000000..c5618870946 --- /dev/null +++ b/test/integration/branchcuts/generated-release-gating-jobs/test2/data/openshift-release-master__ci-4.5-upgrade-from-stable-4.4.yaml @@ -0,0 +1,74 @@ +releases: + initial: + prerelease: + product: ocp + version_bounds: + lower: 4.4.0-0 + upper: 4.5.0-0 + latest: + candidate: + product: ocp + stream: ci + version: "4.5" +resources: + '*': + requests: + cpu: 100m + memory: 200Mi +tests: +- as: e2e-aws-upgrade + interval: 720h + steps: + cluster_profile: aws + workflow: openshift-upgrade-aws +- as: e2e-gcp-upgrade + interval: 720h + steps: + cluster_profile: gcp + workflow: openshift-upgrade-gcp +- as: e2e-azure-upgrade + interval: 720h + steps: + cluster_profile: azure-2 + env: + BASE_DOMAIN: ci2.azure.devcluster.openshift.com + workflow: openshift-upgrade-azure +- as: e2e-ovirt-upgrade + interval: 720h + steps: + cluster_profile: ovirt + env: + BASE_DOMAIN: ci2.azure.devcluster.openshift.com + test: + - as: test + commands: make upgrade_test + credentials: + - mount_path: /var/run/cnv-ci-brew-pull-secret + name: cnv-ci-brew-pull-secret + namespace: test-credentials + env: + - default: '|7820aea2-0d75-11e7-9259-28d244ea5a6d.hhav.f63e13' + name: BREW_IMAGE_REGISTRY_USERNAME + - default: /var/run/cnv-ci-brew-pull-secret/token + name: BREW_IMAGE_REGISTRY_TOKEN_PATH + - default: v0.45.0 + name: KUBEVIRT_RELEASE + - default: "4.5" + name: OCP_VERSION + - default: /tmp/artifacts + name: ARTIFACTS_DIR + - default: openshift-cnv + name: TARGET_NAMESPACE + - default: "true" + name: PRODUCTION_RELEASE + from: cnv-ci-src-upgrade + resources: + requests: + cpu: 100m + memory: 600Mi + workflow: openshift-upgrade-ovirt-release-4.4-4.5 +zz_generated_metadata: + branch: master + org: openshift + repo: release + variant: ci-4.5-upgrade-from-stable-4.4 diff --git a/test/integration/branchcuts/generated-release-gating-jobs/test2/test_config b/test/integration/branchcuts/generated-release-gating-jobs/test2/test_config new file mode 100644 index 00000000000..67efd1b7052 --- /dev/null +++ b/test/integration/branchcuts/generated-release-gating-jobs/test2/test_config @@ -0,0 +1,3 @@ +current_ocp_release=4.5 +future_ocp_release=4.6 +interval=168 \ No newline at end of file diff --git a/test/integration/branchcuts/generated-release-gating-jobs/test2/want/openshift-release-master__ci-4.6-upgrade-from-stable-4.5.yaml b/test/integration/branchcuts/generated-release-gating-jobs/test2/want/openshift-release-master__ci-4.6-upgrade-from-stable-4.5.yaml new file mode 100644 index 00000000000..1695ec20674 --- /dev/null +++ b/test/integration/branchcuts/generated-release-gating-jobs/test2/want/openshift-release-master__ci-4.6-upgrade-from-stable-4.5.yaml @@ -0,0 +1,74 @@ +releases: + initial: + prerelease: + product: ocp + version_bounds: + lower: 4.5.0-0 + upper: 4.6.0-0 + latest: + candidate: + product: ocp + stream: ci + version: "4.6" +resources: + '*': + requests: + cpu: 100m + memory: 200Mi +tests: +- as: e2e-aws-upgrade + interval: 168h + steps: + cluster_profile: aws + workflow: openshift-upgrade-aws +- as: e2e-gcp-upgrade + interval: 168h + steps: + cluster_profile: gcp + workflow: openshift-upgrade-gcp +- as: e2e-azure-upgrade + interval: 168h + steps: + cluster_profile: azure-2 + env: + BASE_DOMAIN: ci2.azure.devcluster.openshift.com + workflow: openshift-upgrade-azure +- as: e2e-ovirt-upgrade + interval: 168h + steps: + cluster_profile: ovirt + env: + BASE_DOMAIN: ci2.azure.devcluster.openshift.com + test: + - as: test + commands: make upgrade_test + credentials: + - mount_path: /var/run/cnv-ci-brew-pull-secret + name: cnv-ci-brew-pull-secret + namespace: test-credentials + env: + - default: '|7820aea2-0d75-11e7-9259-28d244ea5a6d.hhav.f63e13' + name: BREW_IMAGE_REGISTRY_USERNAME + - default: /var/run/cnv-ci-brew-pull-secret/token + name: BREW_IMAGE_REGISTRY_TOKEN_PATH + - default: v0.45.0 + name: KUBEVIRT_RELEASE + - default: "4.6" + name: OCP_VERSION + - default: /tmp/artifacts + name: ARTIFACTS_DIR + - default: openshift-cnv + name: TARGET_NAMESPACE + - default: "true" + name: PRODUCTION_RELEASE + from: cnv-ci-src-upgrade + resources: + requests: + cpu: 100m + memory: 600Mi + workflow: openshift-upgrade-ovirt-release-4.5-4.6 +zz_generated_metadata: + branch: master + org: openshift + repo: release + variant: ci-4.6-upgrade-from-stable-4.5 diff --git a/test/integration/branchcuts/rpm-deps-mirroring-services/test1/data/ocp-4.10-rhel8.repo b/test/integration/branchcuts/rpm-deps-mirroring-services/test1/data/ocp-4.10-rhel8.repo new file mode 100644 index 00000000000..17df7172986 --- /dev/null +++ b/test/integration/branchcuts/rpm-deps-mirroring-services/test1/data/ocp-4.10-rhel8.repo @@ -0,0 +1,100 @@ +[rhel-8-baseos] +name = rhel-8-baseos +baseurl = https://cdn.redhat.com/content/eus/rhel8/8.4/x86_64/baseos/os/ +enabled = 1 +gpgkey = https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-release https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-beta https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-openshifthosted +sslverify = false +gpgcheck = 0 +# https://projects.engineering.redhat.com/browse/RCM-65421 +sslclientkey = /tmp/key/rh-cdn.pem +sslclientcert = /tmp/key/rh-cdn.pem +failovermethod = priority + +[rhel-8-appstream] +name = rhel-8-appstream +baseurl = https://cdn.redhat.com/content/eus/rhel8/8.4/x86_64/appstream/os/ +enabled = 1 +gpgkey = https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-release https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-beta https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-openshifthosted +sslverify = false +gpgcheck = 0 +# https://projects.engineering.redhat.com/browse/RCM-65421 +sslclientkey = /tmp/key/rh-cdn.pem +sslclientcert = /tmp/key/rh-cdn.pem +failovermethod = priority + +[rhel-8-server-ose] +name = rhel-8-server-ose +baseurl = https://mirror2.openshift.com/enterprise/reposync/4.10/rhel-8-server-ose-rpms +sslverify = false +username_file=/tmp/mirror-enterprise-basic-auth/username +password_file=/tmp/mirror-enterprise-basic-auth/password +failovermethod = priority +skip_if_unavailable = true +gpgcheck = 0 +gpgkey = https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-release https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-beta https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-openshifthosted + +[rhel-8-fast-datapath] +name = rhel-8-fast-datapath +baseurl = https://cdn.redhat.com/content/dist/layered/rhel8/x86_64/fast-datapath/os/ +sslverify = false +sslclientkey = /tmp/key/rh-cdn.pem +sslclientcert = /tmp/key/rh-cdn.pem +failovermethod = priority +gpgcheck = 0 +gpgkey = https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-release https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-beta https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-openshifthosted + +[rhel-8-nfv] +name = rhel-8-nfv +baseurl = https://cdn.redhat.com/content/tus/rhel8/8.4/x86_64/nfv/os/ +sslverify = false +sslclientkey = /tmp/key/rh-cdn.pem +sslclientcert = /tmp/key/rh-cdn.pem +failovermethod = priority +gpgcheck = 0 +gpgkey = https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-release https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-beta https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-openshifthosted + +[rhel-8-advanced-virt] +name = rhel-8-advanced-virt +baseurl = https://cdn.redhat.com/content/eus/rhel8/8.4/x86_64/advanced-virt/os/ +sslverify = false +sslclientkey = /tmp/key/rh-cdn.pem +sslclientcert = /tmp/key/rh-cdn.pem +failovermethod = priority +gpgcheck = 0 +gpgkey = https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-release https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-beta https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-openshifthosted + +[rhel-8-ansible-2.9] +name = rhel-8-ansible-2.9 +baseurl = https://cdn.redhat.com/content/dist/layered/rhel8/x86_64/ansible/2.9/os/ +enabled = 1 +gpgkey = https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-release https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-beta https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-openshifthosted +sslverify = false +gpgcheck = 0 +# https://projects.engineering.redhat.com/browse/RCM-65421 +sslclientkey = /tmp/key/rh-cdn.pem +sslclientcert = /tmp/key/rh-cdn.pem +failovermethod = priority + +[openstack-16-for-rhel-8-rpms] +name = openstack-16-for-rhel-8-rpms +baseurl = https://cdn.redhat.com/content/dist/layered/rhel8/x86_64/openstack/16.2/os/ +enabled = 1 +gpgkey = https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-release https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-beta https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-openshifthosted +sslverify = false +gpgcheck = 0 +# https://projects.engineering.redhat.com/browse/RCM-65421 +sslclientkey = /tmp/key/rh-cdn.pem +sslclientcert = /tmp/key/rh-cdn.pem +failovermethod = priority + +[rhel-8-codeready-builder-rpms] +name = rhel-8-codeready-builder-rpms +baseurl = https://cdn.redhat.com/content/eus/rhel8/8.4/x86_64/codeready-builder/os/ +enabled = 1 +gpgkey = https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-release https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-beta https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-openshifthosted +sslverify = false +gpgcheck = 0 +# https://projects.engineering.redhat.com/browse/RCM-65421 +sslclientkey = /tmp/key/rh-cdn.pem +sslclientcert = /tmp/key/rh-cdn.pem +failovermethod = priority diff --git a/test/integration/branchcuts/rpm-deps-mirroring-services/test1/test_config b/test/integration/branchcuts/rpm-deps-mirroring-services/test1/test_config new file mode 100644 index 00000000000..27685bfd7e9 --- /dev/null +++ b/test/integration/branchcuts/rpm-deps-mirroring-services/test1/test_config @@ -0,0 +1,2 @@ +current_ocp_release=4.10 +future_ocp_release=4.11 \ No newline at end of file diff --git a/test/integration/branchcuts/rpm-deps-mirroring-services/test1/want/ocp-4.11-rhel8.repo b/test/integration/branchcuts/rpm-deps-mirroring-services/test1/want/ocp-4.11-rhel8.repo new file mode 100644 index 00000000000..45bb9981ad3 --- /dev/null +++ b/test/integration/branchcuts/rpm-deps-mirroring-services/test1/want/ocp-4.11-rhel8.repo @@ -0,0 +1,100 @@ +[rhel-8-baseos] +name = rhel-8-baseos +baseurl = https://cdn.redhat.com/content/eus/rhel8/8.4/x86_64/baseos/os/ +enabled = 1 +gpgkey = https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-release https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-beta https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-openshifthosted +sslverify = false +gpgcheck = 0 +# https://projects.engineering.redhat.com/browse/RCM-65421 +sslclientkey = /tmp/key/rh-cdn.pem +sslclientcert = /tmp/key/rh-cdn.pem +failovermethod = priority + +[rhel-8-appstream] +name = rhel-8-appstream +baseurl = https://cdn.redhat.com/content/eus/rhel8/8.4/x86_64/appstream/os/ +enabled = 1 +gpgkey = https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-release https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-beta https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-openshifthosted +sslverify = false +gpgcheck = 0 +# https://projects.engineering.redhat.com/browse/RCM-65421 +sslclientkey = /tmp/key/rh-cdn.pem +sslclientcert = /tmp/key/rh-cdn.pem +failovermethod = priority + +[rhel-8-server-ose] +name = rhel-8-server-ose +baseurl = https://mirror2.openshift.com/enterprise/reposync/4.11/rhel-8-server-ose-rpms +sslverify = false +username_file = /tmp/mirror-enterprise-basic-auth/username +password_file = /tmp/mirror-enterprise-basic-auth/password +failovermethod = priority +skip_if_unavailable = true +gpgcheck = 0 +gpgkey = https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-release https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-beta https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-openshifthosted + +[rhel-8-fast-datapath] +name = rhel-8-fast-datapath +baseurl = https://cdn.redhat.com/content/dist/layered/rhel8/x86_64/fast-datapath/os/ +sslverify = false +sslclientkey = /tmp/key/rh-cdn.pem +sslclientcert = /tmp/key/rh-cdn.pem +failovermethod = priority +gpgcheck = 0 +gpgkey = https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-release https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-beta https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-openshifthosted + +[rhel-8-nfv] +name = rhel-8-nfv +baseurl = https://cdn.redhat.com/content/tus/rhel8/8.4/x86_64/nfv/os/ +sslverify = false +sslclientkey = /tmp/key/rh-cdn.pem +sslclientcert = /tmp/key/rh-cdn.pem +failovermethod = priority +gpgcheck = 0 +gpgkey = https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-release https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-beta https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-openshifthosted + +[rhel-8-advanced-virt] +name = rhel-8-advanced-virt +baseurl = https://cdn.redhat.com/content/eus/rhel8/8.4/x86_64/advanced-virt/os/ +sslverify = false +sslclientkey = /tmp/key/rh-cdn.pem +sslclientcert = /tmp/key/rh-cdn.pem +failovermethod = priority +gpgcheck = 0 +gpgkey = https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-release https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-beta https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-openshifthosted + +[rhel-8-ansible-2.9] +name = rhel-8-ansible-2.9 +baseurl = https://cdn.redhat.com/content/dist/layered/rhel8/x86_64/ansible/2.9/os/ +enabled = 1 +gpgkey = https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-release https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-beta https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-openshifthosted +sslverify = false +gpgcheck = 0 +# https://projects.engineering.redhat.com/browse/RCM-65421 +sslclientkey = /tmp/key/rh-cdn.pem +sslclientcert = /tmp/key/rh-cdn.pem +failovermethod = priority + +[openstack-16-for-rhel-8-rpms] +name = openstack-16-for-rhel-8-rpms +baseurl = https://cdn.redhat.com/content/dist/layered/rhel8/x86_64/openstack/16.2/os/ +enabled = 1 +gpgkey = https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-release https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-beta https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-openshifthosted +sslverify = false +gpgcheck = 0 +# https://projects.engineering.redhat.com/browse/RCM-65421 +sslclientkey = /tmp/key/rh-cdn.pem +sslclientcert = /tmp/key/rh-cdn.pem +failovermethod = priority + +[rhel-8-codeready-builder-rpms] +name = rhel-8-codeready-builder-rpms +baseurl = https://cdn.redhat.com/content/eus/rhel8/8.4/x86_64/codeready-builder/os/ +enabled = 1 +gpgkey = https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-release https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-beta https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-openshifthosted +sslverify = false +gpgcheck = 0 +# https://projects.engineering.redhat.com/browse/RCM-65421 +sslclientkey = /tmp/key/rh-cdn.pem +sslclientcert = /tmp/key/rh-cdn.pem +failovermethod = priority