Before branching day: step 1

cmd/branchingconfigmanagers/rpm-deps-mirroring-services/.gitignore

@@ -0,0 +1 @@
+rpm-deps-mirroring-services

cmd/branchingconfigmanagers/rpm-deps-mirroring-services/README.md

@@ -0,0 +1,14 @@
+# RPM dependencies mirroring services
+This manager attempts to automatize step (1.) of "[Few weeks before branching day](https://docs.google.com/document/d/1Z6ejnDCOCvNv9PWkyNPzVbjuLbDMAAT5GEeDpzb0SMs/edit#heading=h.r9xn02r1cyfn)" phase.
+
+## Usage
+### Options:
+- `--current-release` specifies the current OCP version
+- `--release-repo` is the absolution path to `openshift/release` repository
+
+### Example
+```sh
+    $ ./generated-release-gating-jobs \
+        --current-release "4.12" \
+        --release-repo "/full/path/to/openshift/release/repo"
+```

cmd/branchingconfigmanagers/rpm-deps-mirroring-services/main.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"path"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	"gopkg.in/ini.v1"
+
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+
+	"github.com/openshift/ci-tools/pkg/api/ocplifecycle"
+	"github.com/openshift/ci-tools/pkg/branchcuts/bumper"
+)
+
+const (
+	rpmMirroringServicesPath              = "core-services/release-controller/_repos"
+	rpmMirroringServicesGlobPatternFormat = "ocp-%s*.repo"
+)
+
+type options struct {
+	curOCPVersion  string
+	releaseRepoDir string
+	logLevel       int
+}
+
+func gatherOptions() (*options, error) {
+	var errs []error
+	o := &options{}
+	flag.StringVar(&o.curOCPVersion, "current-release", "", "Current OCP version")
+	flag.StringVar(&o.releaseRepoDir, "release-repo", "", "Path to 'openshift/release/ folder")
+	flag.IntVar(&o.logLevel, "log-level", int(logrus.DebugLevel), "Log level")
+	flag.Parse()
+
+	if _, err := ocplifecycle.ParseMajorMinor(o.curOCPVersion); o.curOCPVersion != "" && err != nil {
+		errs = append(errs, fmt.Errorf("error parsing cur-ocp-ver %s", o.curOCPVersion))
+	}
+
+	if o.releaseRepoDir != "" {
+		if !path.IsAbs(o.releaseRepoDir) {
+			errs = append(errs, errors.New("error parsing release repo path: path has to be absolute"))
+		}
+	} else {
+		errs = append(errs, errors.New("error parsing release repo path: path is mandatory"))
+	}
+
+	return o, utilerrors.NewAggregate(errs)
+}
+
+func main() {
+	o, err := gatherOptions()
+	if err != nil {
+		logrus.WithError(err).Fatal("failed to gather options")
+	}
+
+	logrus.SetLevel(logrus.Level(o.logLevel))
+	logrus.Debugf("using options %+v", o)
+
+	if err = reconcile(time.Now(), o); err != nil {
+		logrus.WithError(err).Fatal("failed to reconcile the status")
+	}
+	logrus.Info("status reconciled")
+}
+
+func reconcile(now time.Time, o *options) error {
+	logrus.Debugf("using options %+v", o)
+	bumpOpts := bumper.RepoBumperOptions{
+		FilesDir:      path.Join(o.releaseRepoDir, rpmMirroringServicesPath),
+		GlobPattern:   fmt.Sprintf(rpmMirroringServicesGlobPatternFormat, o.curOCPVersion),
+		CurOCPRelease: o.curOCPVersion,
+	}
+	logrus.Debugf("bumpOpts: %+v", bumpOpts)
+
+	b, err := bumper.NewRepoBumper(&bumpOpts)
+	if err != nil {
+		return fmt.Errorf("new repo bumper: %w", err)
+	}
+
+	if err := bumper.Bump[*ini.File](b, &bumper.BumpingOptions{}); err != nil {
+		return fmt.Errorf("bumper: %w", err)
+	}
+	return nil
+}

go.mod

@@ -194,7 +194,7 @@ require (
 	google.golang.org/grpc v1.47.0 // indirect
 	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/ini.v1 v1.63.2 // indirect
+	gopkg.in/ini.v1 v1.63.2
 	gopkg.in/square/go-jose.v2 v2.3.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/apiextensions-apiserver v0.24.2 // indirect

pkg/branchcuts/bumper/bumper.go

@@ -0,0 +1,110 @@
+package bumper
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/openshift/ci-tools/pkg/util"
+)
+
+type Bumper[T any] interface {
+	GetFiles() ([]string, error)
+
+	Unmarshall(file string) (T, error)
+
+	BumpFilename(filename string, obj T) (string, error)
+
+	BumpContent(obj T) (T, error)
+
+	Marshall(obj T, bumpedFilename, dir string) error
+}
+
+type BumpingOptions struct {
+	OutDir string
+}
+
+// Bump bumps files using the Bumpers b according to the BumpingOptions.
+func Bump[T any](b Bumper[T], o *BumpingOptions) error {
+	filesCh := make(chan string)
+	produce := func() error {
+		defer close(filesCh)
+		files, err := b.GetFiles()
+		logrus.Debugf("files: %+v", files)
+		if err != nil {
+			return err
+		}
+		for _, f := range files {
+			filesCh <- f
+		}
+		return nil
+	}
+	errsChan := make(chan error)
+	map_ := func() error {
+		for f := range filesCh {
+			if err := BumpObject(b, f, o.OutDir); err != nil {
+				errsChan <- err
+			}
+		}
+		return nil
+	}
+	return util.ProduceMap(0, produce, map_, errsChan)
+}
+
+func BumpObject[T any](b Bumper[T], file, outDir string) error {
+	logrus.Infof("bumping config %s", file)
+
+	srcFileFullPath := file
+
+	obj, err := b.Unmarshall(srcFileFullPath)
+	if err != nil {
+		logrus.WithError(err).Errorf("failed to unmarshall file %s", srcFileFullPath)
+		return fmt.Errorf("unmarshall file %s: %w", file, err)
+	}
+
+	filename := path.Base(file)
+
+	logrus.Infof("bumping filename %s", filename)
+	bumpedFilename, err := b.BumpFilename(filename, obj)
+	if err != nil {
+		logrus.WithError(err).Errorf("error bumping file %s", bumpedFilename)
+		return fmt.Errorf("bump filename: %w", err)
+	}
+	logrus.Infof("bumped filename %s", bumpedFilename)
+
+	outDir = getOutDir(file, outDir)
+	logrus.Debugf("out dir: %s", outDir)
+	dstFileFullPath := path.Join(outDir, bumpedFilename)
+
+	if _, err := os.Stat(dstFileFullPath); err == nil {
+		logrus.WithError(err).Warnf("file %s already exists, skipping", dstFileFullPath)
+		return fmt.Errorf("file %s already exists", dstFileFullPath)
+	} else if !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("file exists: %w", err)
+	}
+
+	logrus.Infof("bumping obj")
+	bumpedObj, err := b.BumpContent(obj)
+	if err != nil {
+		logrus.WithError(err).Error("error bumping obj")
+		return fmt.Errorf("bump object: %w", err)
+	}
+
+	logrus.Infof("marshalling obj %s to %s", bumpedFilename, outDir)
+	if err := b.Marshall(bumpedObj, bumpedFilename, outDir); err != nil {
+		logrus.WithError(err).Error("error marshalling2 obj")
+		return fmt.Errorf("marshall obj: %w", err)
+	}
+
+	return nil
+}
+
+func getOutDir(file string, dir string) string {
+	if dir != "" {
+		return dir
+	}
+	return path.Dir(file)
+}

pkg/branchcuts/bumper/repo-bumper.go

@@ -0,0 +1,139 @@
+package bumper
+
+import (
+	"bytes"
+	"fmt"
+	"io/fs"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+
+	"gopkg.in/ini.v1"
+
+	"github.com/openshift/ci-tools/pkg/api/ocplifecycle"
+)
+
+type (
+	RepoBumper struct {
+		GlobPattern string
+		FilesDir    string
+		OCPRelease  ocplifecycle.MajorMinor
+	}
+
+	RepoBumperOptions struct {
+		GlobPattern   string
+		FilesDir      string
+		CurOCPRelease string
+	}
+
+	rhelRepo struct {
+		BaseUrl           string `ini:"baseurl,omitempty"`
+		Enabled           int    `ini:"baseurl,omitempty"`
+		FailoverMethod    bool   `ini:"failovermethod,omitempty"`
+		GPGCheck          int    `ini:"gpgcheck,omitempty"`
+		GPGKey            string `ini:"gpgkey,omitempty"`
+		Name              string `ini:"name,omitempty"`
+		PasswordFile      string `ini:"password_file,omitempty"`
+		SkipIfUnavailable bool   `ini:"skip_if_unavailable,omitempty"`
+		SSLClientCert     string `ini:"sslclientcert,omitempty"`
+		SSLClientKey      string `ini:"sslclientkey,omitempty"`
+		SSLVerify         bool   `ini:"sslverify,omitempty"`
+		UsernameFile      string `ini:"username_file,omitempty"`
+	}
+)
+
+var (
+	majorMinorSeparators = []string{".", "-"}
+
+	_ Bumper[*ini.File] = &RepoBumper{}
+)
+
+func NewRepoBumper(o *RepoBumperOptions) (*RepoBumper, error) {
+	mm, err := ocplifecycle.ParseMajorMinor(o.CurOCPRelease)
+	if err != nil {
+		return nil, fmt.Errorf("Error parsing %s", o.CurOCPRelease)
+	}
+	return &RepoBumper{
+		GlobPattern: o.GlobPattern,
+		FilesDir:    o.FilesDir,
+		OCPRelease:  *mm,
+	}, nil
+}
+
+func (b *RepoBumper) GetFiles() ([]string, error) {
+	dirFs := os.DirFS(b.FilesDir)
+	matches, err := fs.Glob(dirFs, b.GlobPattern)
+	if err != nil {
+		return nil, err
+	}
+	files := make([]string, 0, len(matches))
+	for _, f := range matches {
+		fileFullPath := path.Join(b.FilesDir, f)
+		files = append(files, fileFullPath)
+	}
+	return files, nil
+}
+
+func (b *RepoBumper) Unmarshall(file string) (*ini.File, error) {
+	return ini.Load(file)
+}
+
+func (b *RepoBumper) BumpFilename(filename string, _ *ini.File) (string, error) {
+	curRelease := fmt.Sprintf("%d.%d", b.OCPRelease.Major, b.OCPRelease.Minor)
+	futureRelease := fmt.Sprintf("%d.%d", b.OCPRelease.Major, b.OCPRelease.Minor+1)
+	return strings.ReplaceAll(filename, curRelease, futureRelease), nil
+}
+
+func (b *RepoBumper) BumpContent(file *ini.File) (*ini.File, error) {
+	for _, section := range file.Sections() {
+		repo := rhelRepo{}
+		if err := section.MapTo(&repo); err != nil {
+			return nil, err
+		}
+
+		for _, s := range majorMinorSeparators {
+			curRelease := fmt.Sprintf("%d%s%d", b.OCPRelease.Major, s, b.OCPRelease.Minor)
+			futureRelease := fmt.Sprintf("%d%s%d", b.OCPRelease.Major, s, b.OCPRelease.Minor+1)
+			repo.BaseUrl = strings.ReplaceAll(repo.BaseUrl, curRelease, futureRelease)
+		}
+
+		if err := section.ReflectFrom(&repo); err != nil {
+			return nil, err
+		}
+	}
+	return file, nil
+}
+
+func (b *RepoBumper) Marshall(file *ini.File, bumpedFilename, dir string) error {
+	filePath := path.Join(dir, bumpedFilename)
+	return saveIniFile(filePath, file)
+}
+
+func saveIniFile(path string, f *ini.File) error {
+	ini.PrettySection = true
+	ini.PrettyFormat = false
+	ini.PrettyEqual = true
+
+	// What follow should have be avoided by using f.SaveTo(path) directly,
+	// but unfortunately it appends a double '\n' at the end of the file
+	// that makes it different from the original one: we should only bump the fields of
+	// interest without doing anything else.
+	// Consider opening a PR that fixes this issue, even if I'm not sure this can be
+	// considered an issue.
+	buf := bytes.NewBuffer(nil)
+	if _, err := f.WriteTo(buf); err != nil {
+		return err
+	}
+	bs := buf.Bytes()
+
+	doubleNewLine := ini.LineBreak + ini.LineBreak
+	if strings.HasSuffix(string(bs), doubleNewLine) {
+		bs = bs[0 : len(bs)-len(ini.LineBreak)]
+	}
+	if err := ioutil.WriteFile(path, bs, 0666); err != nil {
+		return err
+	}
+
+	return nil
+}