From fb0b8061296f1f8173b6f84508dbe29e48d9990d Mon Sep 17 00:00:00 2001 From: Danilo Gemoli Date: Tue, 31 May 2022 10:37:12 +0200 Subject: [PATCH] branch-cut: rpm mirroring services --- .../rpm-deps-mirroring-services/.gitignore | 1 + .../rpm-deps-mirroring-services/README.md | 14 ++ .../rpm-deps-mirroring-services/main.go | 86 +++++++++++ go.mod | 2 +- pkg/branchcuts/bumper/bumper.go | 110 ++++++++++++++ pkg/branchcuts/bumper/repo-bumper.go | 139 ++++++++++++++++++ pkg/branchcuts/bumper/repo-bumper_test.go | 80 ++++++++++ 7 files changed, 431 insertions(+), 1 deletion(-) create mode 100644 cmd/branchingconfigmanagers/rpm-deps-mirroring-services/.gitignore create mode 100644 cmd/branchingconfigmanagers/rpm-deps-mirroring-services/README.md create mode 100644 cmd/branchingconfigmanagers/rpm-deps-mirroring-services/main.go create mode 100644 pkg/branchcuts/bumper/bumper.go create mode 100644 pkg/branchcuts/bumper/repo-bumper.go create mode 100644 pkg/branchcuts/bumper/repo-bumper_test.go diff --git a/cmd/branchingconfigmanagers/rpm-deps-mirroring-services/.gitignore b/cmd/branchingconfigmanagers/rpm-deps-mirroring-services/.gitignore new file mode 100644 index 0000000000..e71ba72024 --- /dev/null +++ b/cmd/branchingconfigmanagers/rpm-deps-mirroring-services/.gitignore @@ -0,0 +1 @@ +rpm-deps-mirroring-services \ No newline at end of file diff --git a/cmd/branchingconfigmanagers/rpm-deps-mirroring-services/README.md b/cmd/branchingconfigmanagers/rpm-deps-mirroring-services/README.md new file mode 100644 index 0000000000..67ef87b4fc --- /dev/null +++ b/cmd/branchingconfigmanagers/rpm-deps-mirroring-services/README.md @@ -0,0 +1,14 @@ +# RPM dependencies mirroring services +This manager attempts to automatize step (1.) of "[Few weeks before branching day](https://docs.google.com/document/d/1Z6ejnDCOCvNv9PWkyNPzVbjuLbDMAAT5GEeDpzb0SMs/edit#heading=h.r9xn02r1cyfn)" phase. + +## Usage +### Options: +- `--current-release` specifies the current OCP version +- `--release-repo` is the absolution path to `openshift/release` repository + +### Example +```sh + $ ./generated-release-gating-jobs \ + --current-release "4.12" \ + --release-repo "/full/path/to/openshift/release/repo" +``` \ No newline at end of file diff --git a/cmd/branchingconfigmanagers/rpm-deps-mirroring-services/main.go b/cmd/branchingconfigmanagers/rpm-deps-mirroring-services/main.go new file mode 100644 index 0000000000..9018496533 --- /dev/null +++ b/cmd/branchingconfigmanagers/rpm-deps-mirroring-services/main.go @@ -0,0 +1,86 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "path" + "time" + + "github.com/sirupsen/logrus" + "gopkg.in/ini.v1" + + utilerrors "k8s.io/apimachinery/pkg/util/errors" + + "github.com/openshift/ci-tools/pkg/api/ocplifecycle" + "github.com/openshift/ci-tools/pkg/branchcuts/bumper" +) + +const ( + rpmMirroringServicesPath = "core-services/release-controller/_repos" + rpmMirroringServicesGlobPatternFormat = "ocp-%s*.repo" +) + +type options struct { + curOCPVersion string + releaseRepoDir string + logLevel int +} + +func gatherOptions() (*options, error) { + var errs []error + o := &options{} + flag.StringVar(&o.curOCPVersion, "current-release", "", "Current OCP version") + flag.StringVar(&o.releaseRepoDir, "release-repo", "", "Path to 'openshift/release/ folder") + flag.IntVar(&o.logLevel, "log-level", int(logrus.DebugLevel), "Log level") + flag.Parse() + + if _, err := ocplifecycle.ParseMajorMinor(o.curOCPVersion); o.curOCPVersion != "" && err != nil { + errs = append(errs, fmt.Errorf("error parsing cur-ocp-ver %s", o.curOCPVersion)) + } + + if o.releaseRepoDir != "" { + if !path.IsAbs(o.releaseRepoDir) { + errs = append(errs, errors.New("error parsing release repo path: path has to be absolute")) + } + } else { + errs = append(errs, errors.New("error parsing release repo path: path is mandatory")) + } + + return o, utilerrors.NewAggregate(errs) +} + +func main() { + o, err := gatherOptions() + if err != nil { + logrus.WithError(err).Fatal("failed to gather options") + } + + logrus.SetLevel(logrus.Level(o.logLevel)) + logrus.Debugf("using options %+v", o) + + if err = reconcile(time.Now(), o); err != nil { + logrus.WithError(err).Fatal("failed to reconcile the status") + } + logrus.Info("status reconciled") +} + +func reconcile(now time.Time, o *options) error { + logrus.Debugf("using options %+v", o) + bumpOpts := bumper.RepoBumperOptions{ + FilesDir: path.Join(o.releaseRepoDir, rpmMirroringServicesPath), + GlobPattern: fmt.Sprintf(rpmMirroringServicesGlobPatternFormat, o.curOCPVersion), + CurOCPRelease: o.curOCPVersion, + } + logrus.Debugf("bumpOpts: %+v", bumpOpts) + + b, err := bumper.NewRepoBumper(&bumpOpts) + if err != nil { + return fmt.Errorf("new repo bumper: %w", err) + } + + if err := bumper.Bump[*ini.File](b, &bumper.BumpingOptions{}); err != nil { + return fmt.Errorf("bumper: %w", err) + } + return nil +} diff --git a/go.mod b/go.mod index d67b33b5c8..98b64d47af 100644 --- a/go.mod +++ b/go.mod @@ -194,7 +194,7 @@ require ( google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.63.2 // indirect + gopkg.in/ini.v1 v1.63.2 gopkg.in/square/go-jose.v2 v2.3.1 // indirect gopkg.in/yaml.v3 v3.0.1 k8s.io/apiextensions-apiserver v0.24.2 // indirect diff --git a/pkg/branchcuts/bumper/bumper.go b/pkg/branchcuts/bumper/bumper.go new file mode 100644 index 0000000000..7bf193d37a --- /dev/null +++ b/pkg/branchcuts/bumper/bumper.go @@ -0,0 +1,110 @@ +package bumper + +import ( + "errors" + "fmt" + "os" + "path" + + "github.com/sirupsen/logrus" + + "github.com/openshift/ci-tools/pkg/util" +) + +type Bumper[T any] interface { + GetFiles() ([]string, error) + + Unmarshall(file string) (T, error) + + BumpFilename(filename string, obj T) (string, error) + + BumpContent(obj T) (T, error) + + Marshall(obj T, bumpedFilename, dir string) error +} + +type BumpingOptions struct { + OutDir string +} + +// Bump bumps files using the Bumpers b according to the BumpingOptions. +func Bump[T any](b Bumper[T], o *BumpingOptions) error { + filesCh := make(chan string) + produce := func() error { + defer close(filesCh) + files, err := b.GetFiles() + logrus.Debugf("files: %+v", files) + if err != nil { + return err + } + for _, f := range files { + filesCh <- f + } + return nil + } + errsChan := make(chan error) + map_ := func() error { + for f := range filesCh { + if err := BumpObject(b, f, o.OutDir); err != nil { + errsChan <- err + } + } + return nil + } + return util.ProduceMap(0, produce, map_, errsChan) +} + +func BumpObject[T any](b Bumper[T], file, outDir string) error { + logrus.Infof("bumping config %s", file) + + srcFileFullPath := file + + obj, err := b.Unmarshall(srcFileFullPath) + if err != nil { + logrus.WithError(err).Errorf("failed to unmarshall file %s", srcFileFullPath) + return fmt.Errorf("unmarshall file %s: %w", file, err) + } + + filename := path.Base(file) + + logrus.Infof("bumping filename %s", filename) + bumpedFilename, err := b.BumpFilename(filename, obj) + if err != nil { + logrus.WithError(err).Errorf("error bumping file %s", bumpedFilename) + return fmt.Errorf("bump filename: %w", err) + } + logrus.Infof("bumped filename %s", bumpedFilename) + + outDir = getOutDir(file, outDir) + logrus.Debugf("out dir: %s", outDir) + dstFileFullPath := path.Join(outDir, bumpedFilename) + + if _, err := os.Stat(dstFileFullPath); err == nil { + logrus.WithError(err).Warnf("file %s already exists, skipping", dstFileFullPath) + return fmt.Errorf("file %s already exists", dstFileFullPath) + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("file exists: %w", err) + } + + logrus.Infof("bumping obj") + bumpedObj, err := b.BumpContent(obj) + if err != nil { + logrus.WithError(err).Error("error bumping obj") + return fmt.Errorf("bump object: %w", err) + } + + logrus.Infof("marshalling obj %s to %s", bumpedFilename, outDir) + if err := b.Marshall(bumpedObj, bumpedFilename, outDir); err != nil { + logrus.WithError(err).Error("error marshalling2 obj") + return fmt.Errorf("marshall obj: %w", err) + } + + return nil +} + +func getOutDir(file string, dir string) string { + if dir != "" { + return dir + } + return path.Dir(file) +} diff --git a/pkg/branchcuts/bumper/repo-bumper.go b/pkg/branchcuts/bumper/repo-bumper.go new file mode 100644 index 0000000000..0e16aa9e82 --- /dev/null +++ b/pkg/branchcuts/bumper/repo-bumper.go @@ -0,0 +1,139 @@ +package bumper + +import ( + "bytes" + "fmt" + "io/fs" + "io/ioutil" + "os" + "path" + "strings" + + "gopkg.in/ini.v1" + + "github.com/openshift/ci-tools/pkg/api/ocplifecycle" +) + +type ( + RepoBumper struct { + GlobPattern string + FilesDir string + OCPRelease ocplifecycle.MajorMinor + } + + RepoBumperOptions struct { + GlobPattern string + FilesDir string + CurOCPRelease string + } + + rhelRepo struct { + BaseUrl string `ini:"baseurl,omitempty"` + Enabled int `ini:"baseurl,omitempty"` + FailoverMethod bool `ini:"failovermethod,omitempty"` + GPGCheck int `ini:"gpgcheck,omitempty"` + GPGKey string `ini:"gpgkey,omitempty"` + Name string `ini:"name,omitempty"` + PasswordFile string `ini:"password_file,omitempty"` + SkipIfUnavailable bool `ini:"skip_if_unavailable,omitempty"` + SSLClientCert string `ini:"sslclientcert,omitempty"` + SSLClientKey string `ini:"sslclientkey,omitempty"` + SSLVerify bool `ini:"sslverify,omitempty"` + UsernameFile string `ini:"username_file,omitempty"` + } +) + +var ( + majorMinorSeparators = []string{".", "-"} + + _ Bumper[*ini.File] = &RepoBumper{} +) + +func NewRepoBumper(o *RepoBumperOptions) (*RepoBumper, error) { + mm, err := ocplifecycle.ParseMajorMinor(o.CurOCPRelease) + if err != nil { + return nil, fmt.Errorf("Error parsing %s", o.CurOCPRelease) + } + return &RepoBumper{ + GlobPattern: o.GlobPattern, + FilesDir: o.FilesDir, + OCPRelease: *mm, + }, nil +} + +func (b *RepoBumper) GetFiles() ([]string, error) { + dirFs := os.DirFS(b.FilesDir) + matches, err := fs.Glob(dirFs, b.GlobPattern) + if err != nil { + return nil, err + } + files := make([]string, 0, len(matches)) + for _, f := range matches { + fileFullPath := path.Join(b.FilesDir, f) + files = append(files, fileFullPath) + } + return files, nil +} + +func (b *RepoBumper) Unmarshall(file string) (*ini.File, error) { + return ini.Load(file) +} + +func (b *RepoBumper) BumpFilename(filename string, _ *ini.File) (string, error) { + curRelease := fmt.Sprintf("%d.%d", b.OCPRelease.Major, b.OCPRelease.Minor) + futureRelease := fmt.Sprintf("%d.%d", b.OCPRelease.Major, b.OCPRelease.Minor+1) + return strings.ReplaceAll(filename, curRelease, futureRelease), nil +} + +func (b *RepoBumper) BumpContent(file *ini.File) (*ini.File, error) { + for _, section := range file.Sections() { + repo := rhelRepo{} + if err := section.MapTo(&repo); err != nil { + return nil, err + } + + for _, s := range majorMinorSeparators { + curRelease := fmt.Sprintf("%d%s%d", b.OCPRelease.Major, s, b.OCPRelease.Minor) + futureRelease := fmt.Sprintf("%d%s%d", b.OCPRelease.Major, s, b.OCPRelease.Minor+1) + repo.BaseUrl = strings.ReplaceAll(repo.BaseUrl, curRelease, futureRelease) + } + + if err := section.ReflectFrom(&repo); err != nil { + return nil, err + } + } + return file, nil +} + +func (b *RepoBumper) Marshall(file *ini.File, bumpedFilename, dir string) error { + filePath := path.Join(dir, bumpedFilename) + return saveIniFile(filePath, file) +} + +func saveIniFile(path string, f *ini.File) error { + ini.PrettySection = true + ini.PrettyFormat = false + ini.PrettyEqual = true + + // What follow should have be avoided by using f.SaveTo(path) directly, + // but unfortunately it appends a double '\n' at the end of the file + // that makes it different from the original one: we should only bump the fields of + // interest without doing anything else. + // Consider opening a PR that fixes this issue, even if I'm not sure this can be + // considered an issue. + buf := bytes.NewBuffer(nil) + if _, err := f.WriteTo(buf); err != nil { + return err + } + bs := buf.Bytes() + + doubleNewLine := ini.LineBreak + ini.LineBreak + if strings.HasSuffix(string(bs), doubleNewLine) { + bs = bs[0 : len(bs)-len(ini.LineBreak)] + } + if err := ioutil.WriteFile(path, bs, 0666); err != nil { + return err + } + + return nil +} diff --git a/pkg/branchcuts/bumper/repo-bumper_test.go b/pkg/branchcuts/bumper/repo-bumper_test.go new file mode 100644 index 0000000000..066543a9f2 --- /dev/null +++ b/pkg/branchcuts/bumper/repo-bumper_test.go @@ -0,0 +1,80 @@ +package bumper_test + +import ( + "strings" + "testing" + + "gopkg.in/ini.v1" + + "github.com/openshift/ci-tools/pkg/branchcuts/bumper" +) + +func TestBumpRepo(t *testing.T) { + tests := []struct { + id string + content string + wantContent string + curOCPRelease string + }{ + { + "Bump properly", + `[rhel-repo-1] +name = test +baseurl = https://fake-repo.com/ocp/4.10/test +`, + `[rhel-repo-1] +name = test +baseurl = https://fake-repo.com/ocp/4.11/test + +`, + "4.10", + }, + { + "Bump dash", + `[rhel-repo-1] +name = test +baseurl = https://fake-repo.com/ocp/4-10/test +`, + `[rhel-repo-1] +name = test +baseurl = https://fake-repo.com/ocp/4-11/test + +`, + "4.10", + }, + } + + ini.PrettySection = true + ini.PrettyFormat = false + ini.PrettyEqual = true + + for _, test := range tests { + t.Run(test.id, func(t *testing.T) { + t.Parallel() + b, err := bumper.NewRepoBumper(&bumper.RepoBumperOptions{ + CurOCPRelease: test.curOCPRelease, + }) + if err != nil { + t.Error(err) + } + file, err := ini.Load([]byte(test.content)) + if err != nil { + t.Error(err) + } + + result, err := b.BumpContent(file) + if err != nil { + t.Error(err) + } else { + buf := strings.Builder{} + if _, err := result.WriteTo(&buf); err != nil { + t.Error(err) + } + resultContent := buf.String() + if test.wantContent != resultContent { + t.Errorf("Expected '%s' but got '%s'\n", test.wantContent, resultContent) + } + } + }) + } +}