From 9e79866007cbef074a93af926ecebd7ca1fa6c1f Mon Sep 17 00:00:00 2001 From: Miguel Elias dos Santos Date: Sat, 26 Mar 2022 10:46:54 +1100 Subject: [PATCH 1/3] Add edge case handling for Repositories.GetContents() --- gen.go | 211 ----------------------------- main.go | 100 ++++++++++++++ src/gen/gen.go | 130 ++++++++++++++++++ src/gen/gen_mutations.go | 21 +++ gen_test.go => src/gen/gen_test.go | 12 +- src/mock/endpointpattern.go | 8 +- src/mock/endpointpattern_test.go | 72 ++++++++++ 7 files changed, 337 insertions(+), 217 deletions(-) delete mode 100644 gen.go create mode 100644 main.go create mode 100644 src/gen/gen.go create mode 100644 src/gen/gen_mutations.go rename gen_test.go => src/gen/gen_test.go (82%) create mode 100644 src/mock/endpointpattern_test.go diff --git a/gen.go b/gen.go deleted file mode 100644 index 3ccc411..0000000 --- a/gen.go +++ /dev/null @@ -1,211 +0,0 @@ -package main - -import ( - "bytes" - "flag" - "fmt" - "io" - "io/ioutil" - "net/http" - "os" - "os/exec" - - "strings" - - "github.com/buger/jsonparser" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" -) - -//go:generate go run gen.go - -const GITHUB_OPENAPI_DEFINITION_LOCATION = "https://github.com/github/rest-api-description/blob/main/descriptions/api.github.com/api.github.com.json?raw=true" - -const OUTPUT_FILE_HEADER = `package mock - -// Code generated by gen.go; DO NOT EDIT. - -` -const OUTPUT_FILEPATH = "src/mock/endpointpattern.go" - -type ScrapeResult struct { - HTTPMethod string - EndpointPattern string -} - -var debug bool - -func init() { - flag.BoolVar(&debug, "debug", false, "output debug information") -} - -func fetchAPIDefinition(l log.Logger) []byte { - resp, err := http.Get(GITHUB_OPENAPI_DEFINITION_LOCATION) - - if err != nil { - level.Error(l).Log( - "msg", "error fetching github's api definition", - "err", err.Error(), - ) - - os.Exit(1) - } - - defer resp.Body.Close() - - bodyBytes, err := io.ReadAll(resp.Body) - - if err != nil { - level.Error(l).Log( - "msg", "error fetching github's api definition", - "err", err.Error(), - ) - - os.Exit(1) - } - - return bodyBytes -} - -// formatToGolangVarName generated the proper golang variable name -// given a endpoint format from the API -func formatToGolangVarName(l log.Logger, sr ScrapeResult) string { - result := strings.Title(strings.ToLower(sr.HTTPMethod)) - - if sr.EndpointPattern == "/" { - return result + "Slash" - } - - // handles urls with dashes in them - pattern := strings.ReplaceAll(sr.EndpointPattern, "-", "/") - - epSplit := strings.Split( - pattern, - "/", - ) - - // handle the first part of the variable name - for _, part := range epSplit { - if len(part) < 1 || string(part[0]) == "{" { - continue - } - - splitPart := strings.Split(part, "_") - - for _, p := range splitPart { - result = result + strings.Title(p) - } - } - - //handle the "By`X`" part of the variable name - for _, part := range epSplit { - if len(part) < 1 { - continue - } - - if string(part[0]) == "{" { - part = strings.ReplaceAll(part, "{", "") - part = strings.ReplaceAll(part, "}", "") - - result += "By" - - for _, splitPart := range strings.Split(part, "_") { - result += strings.Title(splitPart) - } - } - } - - return result -} - -func formatToGolangVarNameAndValue(l log.Logger, lsr ScrapeResult) string { - return fmt.Sprintf( - `var %s EndpointPattern = EndpointPattern{ - Pattern: "%s", - Method: "%s", -} -`, - formatToGolangVarName(l, lsr), - lsr.EndpointPattern, - strings.ToUpper(lsr.HTTPMethod), - ) + "\n" -} - -func main() { - flag.Parse() - - var l log.Logger - - l = log.NewLogfmtLogger(log.NewSyncWriter(os.Stdout)) - - l = log.With(l, "caller", log.DefaultCaller) - - if debug { - l = level.NewFilter(l, level.AllowDebug()) - level.Debug(l).Log("msg", "running in debug mode") - } else { - l = level.NewFilter(l, level.AllowInfo()) - } - - apiDefinition := fetchAPIDefinition(l) - - buf := bytes.NewBuffer([]byte(OUTPUT_FILE_HEADER)) - - jsonparser.ObjectEach( - apiDefinition, - func(key, endpointDefinition []byte, _ jsonparser.ValueType, _ int) error { - endpointPattern := string(key) - - httpMethods := []string{} - - jsonparser.ObjectEach( - endpointDefinition, - func(key, _ []byte, _ jsonparser.ValueType, _ int) error { - httpMethods = append(httpMethods, string(key)) - - return nil - }, - ) - - for _, httpMethod := range httpMethods { - code := formatToGolangVarNameAndValue( - l, - ScrapeResult{ - HTTPMethod: httpMethod, - EndpointPattern: endpointPattern, - }, - ) - - buf.WriteString(code) - } - - return nil - }, - "paths", - ) - - ioutil.WriteFile( - OUTPUT_FILEPATH, - buf.Bytes(), - 0755, - ) - - errorsFound := false - - // to catch possible format errors - if err := exec.Command("gofmt", "-w", "src/mock/endpointpattern.go").Run(); err != nil { - level.Error(l).Log("msg", fmt.Sprintf("error executing gofmt: %s", err.Error())) - errorsFound = true - } - - // to catch everything else (hopefully) - if err := exec.Command("go", "vet", "./...").Run(); err != nil { - level.Error(l).Log("msg", fmt.Sprintf("error executing go vet: %s", err.Error())) - errorsFound = true - } - - if errorsFound { - os.Exit(1) - } -} diff --git a/main.go b/main.go new file mode 100644 index 0000000..0304ca1 --- /dev/null +++ b/main.go @@ -0,0 +1,100 @@ +package main + +import ( + "bytes" + "flag" + "fmt" + "io/ioutil" + "os" + "os/exec" + + "github.com/buger/jsonparser" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/migueleliasweb/go-github-mock/src/gen" +) + +var debug bool + +func init() { + flag.BoolVar(&debug, "debug", false, "output debug information") +} + +func main() { + flag.Parse() + + var l log.Logger + + l = log.NewLogfmtLogger(log.NewSyncWriter(os.Stdout)) + + l = log.With(l, "caller", log.DefaultCaller) + + if debug { + l = level.NewFilter(l, level.AllowDebug()) + level.Debug(l).Log("msg", "running in debug mode") + } else { + l = level.NewFilter(l, level.AllowInfo()) + } + + apiDefinition := gen.FetchAPIDefinition(l) + + buf := bytes.NewBuffer([]byte(gen.OUTPUT_FILE_HEADER)) + + jsonparser.ObjectEach( + apiDefinition, + func(key, endpointDefinition []byte, _ jsonparser.ValueType, _ int) error { + endpointPattern := string(key) + + httpMethods := []string{} + + jsonparser.ObjectEach( + endpointDefinition, + func(key, _ []byte, _ jsonparser.ValueType, _ int) error { + httpMethods = append(httpMethods, string(key)) + + return nil + }, + ) + + for _, httpMethod := range httpMethods { + code := gen.FormatToGolangVarNameAndValue( + l, + gen.ScrapeResult{ + HTTPMethod: httpMethod, + EndpointPattern: endpointPattern, + }, + ) + + buf.WriteString(code) + } + + return nil + }, + "paths", + ) + + ioutil.WriteFile( + gen.OUTPUT_FILEPATH, + buf.Bytes(), + 0755, + ) + + errorsFound := false + + // to catch possible format errors + if err := exec.Command("gofmt", "-w", "src/mock/endpointpattern.go").Run(); err != nil { + level.Error(l).Log("msg", fmt.Sprintf("error executing gofmt: %s", err.Error())) + errorsFound = true + } + + // to catch everything else (hopefully) + if err := exec.Command("go", "vet", "./...").Run(); err != nil { + level.Error(l).Log("msg", fmt.Sprintf("error executing go vet: %s", err.Error())) + errorsFound = true + } + + if errorsFound { + os.Exit(1) + } +} diff --git a/src/gen/gen.go b/src/gen/gen.go new file mode 100644 index 0000000..1e7cefb --- /dev/null +++ b/src/gen/gen.go @@ -0,0 +1,130 @@ +package gen + +import ( + "fmt" + "io" + "net/http" + "os" + "regexp" + "strings" + + "github.com/go-kit/log" + + "github.com/go-kit/log/level" +) + +//go:generate go run gen.go + +const GITHUB_OPENAPI_DEFINITION_LOCATION = "https://github.com/github/rest-api-description/blob/main/descriptions/api.github.com/api.github.com.json?raw=true" + +const OUTPUT_FILE_HEADER = `package mock + +// Code generated; DO NOT EDIT. + +` +const OUTPUT_FILEPATH = "src/mock/endpointpattern.go" + +type ScrapeResult struct { + HTTPMethod string + EndpointPattern string +} + +func FetchAPIDefinition(l log.Logger) []byte { + resp, err := http.Get(GITHUB_OPENAPI_DEFINITION_LOCATION) + + if err != nil { + level.Error(l).Log( + "msg", "error fetching github's api definition", + "err", err.Error(), + ) + + os.Exit(1) + } + + defer resp.Body.Close() + + bodyBytes, err := io.ReadAll(resp.Body) + + if err != nil { + level.Error(l).Log( + "msg", "error fetching github's api definition", + "err", err.Error(), + ) + + os.Exit(1) + } + + return bodyBytes +} + +// FormatToGolangVarName generated the proper golang variable name +// given a endpoint format from the API +func FormatToGolangVarName(l log.Logger, sr ScrapeResult) string { + result := strings.Title(strings.ToLower(sr.HTTPMethod)) + + if sr.EndpointPattern == "/" { + return result + "Slash" + } + + // handles urls with dashes in them + pattern := strings.ReplaceAll(sr.EndpointPattern, "-", "/") + + // cleans up varname when pattern was mutated + // e.g see `GetReposContentsByOwnerByRepoByPath` + re := regexp.MustCompile(`[a-zA-Z0-9\/\{\}\_]+`) + matches := re.FindAllString(pattern, -1) + pattern = strings.Join(matches, "") + + epSplit := strings.Split( + pattern, + "/", + ) + + // handle the first part of the variable name + for _, part := range epSplit { + if len(part) < 1 || string(part[0]) == "{" { + continue + } + + splitPart := strings.Split(part, "_") + + for _, p := range splitPart { + result = result + strings.Title(p) + } + } + + //handle the "By`X`" part of the variable name + for _, part := range epSplit { + if len(part) < 1 { + continue + } + + if string(part[0]) == "{" { + part = strings.ReplaceAll(part, "{", "") + part = strings.ReplaceAll(part, "}", "") + + result += "By" + + for _, splitPart := range strings.Split(part, "_") { + result += strings.Title(splitPart) + } + } + } + + return result +} + +func FormatToGolangVarNameAndValue(l log.Logger, sr ScrapeResult) string { + sr = applyMutation(sr) + + return fmt.Sprintf( + `var %s EndpointPattern = EndpointPattern{ + Pattern: "%s", + Method: "%s", +} +`, + FormatToGolangVarName(l, sr), + sr.EndpointPattern, + strings.ToUpper(sr.HTTPMethod), + ) + "\n" +} diff --git a/src/gen/gen_mutations.go b/src/gen/gen_mutations.go new file mode 100644 index 0000000..c5ddbf4 --- /dev/null +++ b/src/gen/gen_mutations.go @@ -0,0 +1,21 @@ +package gen + +var enabledMutators = map[string]func(ScrapeResult) ScrapeResult{ + "/repos/{owner}/{repo}/contents/{path}": func(sr ScrapeResult) ScrapeResult { + sr.EndpointPattern = "/repos/{owner}/{repo}/contents/{path:.+}" + + return sr + }, +} + +// applyMutation applies mutation to the scrape result if necessary. +// +// There are some edge cases due to inconsistencies between GitHub's OpenAPI definition +// compared to the real world. +func applyMutation(sr ScrapeResult) ScrapeResult { + if mutator, found := enabledMutators[sr.EndpointPattern]; found { + return mutator(sr) + } + + return sr +} diff --git a/gen_test.go b/src/gen/gen_test.go similarity index 82% rename from gen_test.go rename to src/gen/gen_test.go index 06321fa..695291b 100644 --- a/gen_test.go +++ b/src/gen/gen_test.go @@ -1,4 +1,4 @@ -package main +package gen import ( "testing" @@ -52,10 +52,18 @@ func TestFormatToGolangVarName(t *testing.T) { }, want: "GetReposActionsRunsPendingDeploymentsByOwnerByRepoByRunId", }, + { + name: "withUrlWithNumber", + sr: ScrapeResult{ + EndpointPattern: "/repos/{owner}/{repo}/actions/runs/{run_id}/pending_deployments", + HTTPMethod: "GET", + }, + want: "GetReposActionsRunsPendingDeploymentsByOwnerByRepoByRunId", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := formatToGolangVarName(log.NewNopLogger(), tt.sr); got != tt.want { + if got := FormatToGolangVarName(log.NewNopLogger(), tt.sr); got != tt.want { t.Errorf("formatToGolangVarName() = %v, want %v", got, tt.want) } }) diff --git a/src/mock/endpointpattern.go b/src/mock/endpointpattern.go index da8ef2e..49bb5e9 100644 --- a/src/mock/endpointpattern.go +++ b/src/mock/endpointpattern.go @@ -1,6 +1,6 @@ package mock -// Code generated by gen.go; DO NOT EDIT. +// Code generated; DO NOT EDIT. var GetSlash EndpointPattern = EndpointPattern{ Pattern: "/", @@ -2283,17 +2283,17 @@ var GetReposCompareByOwnerByRepoByBasehead EndpointPattern = EndpointPattern{ } var GetReposContentsByOwnerByRepoByPath EndpointPattern = EndpointPattern{ - Pattern: "/repos/{owner}/{repo}/contents/{path}", + Pattern: "/repos/{owner}/{repo}/contents/{path:.+}", Method: "GET", } var PutReposContentsByOwnerByRepoByPath EndpointPattern = EndpointPattern{ - Pattern: "/repos/{owner}/{repo}/contents/{path}", + Pattern: "/repos/{owner}/{repo}/contents/{path:.+}", Method: "PUT", } var DeleteReposContentsByOwnerByRepoByPath EndpointPattern = EndpointPattern{ - Pattern: "/repos/{owner}/{repo}/contents/{path}", + Pattern: "/repos/{owner}/{repo}/contents/{path:.+}", Method: "DELETE", } diff --git a/src/mock/endpointpattern_test.go b/src/mock/endpointpattern_test.go new file mode 100644 index 0000000..60e1173 --- /dev/null +++ b/src/mock/endpointpattern_test.go @@ -0,0 +1,72 @@ +package mock + +import ( + "context" + "testing" + + "github.com/google/go-github/v41/github" +) + +func TestRepoGetContents2(t *testing.T) { + cases := []struct { + name string + repositoryContent github.RepositoryContent + }{ + { + name: "fileWithoutForwardSlash", + repositoryContent: github.RepositoryContent{ + Encoding: github.String("base64"), + Path: github.String("README.md"), + Content: github.String("fake-content"), + }, + }, + { + name: "fileWithForwardSlash", + repositoryContent: github.RepositoryContent{ + Encoding: github.String("base64"), + Path: github.String("path/test-file.txt"), + Content: github.String("fake-content"), + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(rc github.RepositoryContent) func(t *testing.T) { + return func(t *testing.T) { + mockedHTTPClient := NewMockedHTTPClient( + WithRequestMatch( + GetReposContentsByOwnerByRepoByPath, + rc, + ), + ) + + c := github.NewClient(mockedHTTPClient) + + ctx := context.Background() + + fileContent, _, _, err := c.Repositories.GetContents( + ctx, + "foo", + "bar", + *rc.Path, + &github.RepositoryContentGetOptions{}, + ) + + if *(fileContent.Content) != *rc.Content { + t.Errorf( + "fileContent.Content is %s, want %s", + *(fileContent.Content), + *rc.Content, + ) + } + + if err != nil { + t.Errorf( + "err is %s, want nil", + err.Error(), + ) + } + } + }(c.repositoryContent)) + } +} From f568ce7729ef3e7f61aa464e2d31b848080dd1ad Mon Sep 17 00:00:00 2001 From: Miguel Elias dos Santos Date: Wed, 6 Apr 2022 21:36:32 +1000 Subject: [PATCH 2/3] Add handling for reposGit edge cases --- src/gen/gen.go | 2 -- src/gen/gen_mutations.go | 29 ++++++++++++++++++++++++++--- src/mock/endpointpattern.go | 11 ++++++++--- 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/src/gen/gen.go b/src/gen/gen.go index 1e7cefb..4a2f5ba 100644 --- a/src/gen/gen.go +++ b/src/gen/gen.go @@ -13,8 +13,6 @@ import ( "github.com/go-kit/log/level" ) -//go:generate go run gen.go - const GITHUB_OPENAPI_DEFINITION_LOCATION = "https://github.com/github/rest-api-description/blob/main/descriptions/api.github.com/api.github.com.json?raw=true" const OUTPUT_FILE_HEADER = `package mock diff --git a/src/gen/gen_mutations.go b/src/gen/gen_mutations.go index c5ddbf4..78e8f59 100644 --- a/src/gen/gen_mutations.go +++ b/src/gen/gen_mutations.go @@ -1,11 +1,34 @@ package gen +import ( + "fmt" + "regexp" + "strings" +) + var enabledMutators = map[string]func(ScrapeResult) ScrapeResult{ - "/repos/{owner}/{repo}/contents/{path}": func(sr ScrapeResult) ScrapeResult { - sr.EndpointPattern = "/repos/{owner}/{repo}/contents/{path:.+}" + "/repos/{owner}/{repo}/contents/{path}": allowExtendedLastParamMutatorHelper(), + "/repos/{owner}/{repo}/git/ref/{ref}": allowExtendedLastParamMutatorHelper(), + "/repos/{owner}/{repo}/git/refs/{ref}": allowExtendedLastParamMutatorHelper(), //thanks for the consistency, GitHub +} + +// allowExtendedLastParamMutatorHelper mutates the last param of the endpoint pattern +// allowing it to have any characters (including slashes) +func allowExtendedLastParamMutatorHelper() func(ScrapeResult) ScrapeResult { + return func(sr ScrapeResult) ScrapeResult { + endpointSplits := strings.Split(sr.EndpointPattern, "/") + lastParam := endpointSplits[len(endpointSplits)-1] + + r := regexp.MustCompile(`[a-z]+`) + + lastParamCleaned := r.FindString(lastParam) + + endpointSplits[len(endpointSplits)-1] = fmt.Sprintf("{%s:.+}", lastParamCleaned) + + sr.EndpointPattern = strings.Join(endpointSplits, "/") return sr - }, + } } // applyMutation applies mutation to the scrape result if necessary. diff --git a/src/mock/endpointpattern.go b/src/mock/endpointpattern.go index 49bb5e9..f78e83b 100644 --- a/src/mock/endpointpattern.go +++ b/src/mock/endpointpattern.go @@ -2327,6 +2327,11 @@ var DeleteReposDependabotSecretsByOwnerByRepoBySecretName EndpointPattern = Endp Method: "DELETE", } +var GetReposDependencyGraphCompareByOwnerByRepoByBasehead EndpointPattern = EndpointPattern{ + Pattern: "/repos/{owner}/{repo}/dependency-graph/compare/{basehead}", + Method: "GET", +} + var GetReposDeploymentsByOwnerByRepo EndpointPattern = EndpointPattern{ Pattern: "/repos/{owner}/{repo}/deployments", Method: "GET", @@ -2428,7 +2433,7 @@ var GetReposGitMatchingRefsByOwnerByRepoByRef EndpointPattern = EndpointPattern{ } var GetReposGitRefByOwnerByRepoByRef EndpointPattern = EndpointPattern{ - Pattern: "/repos/{owner}/{repo}/git/ref/{ref}", + Pattern: "/repos/{owner}/{repo}/git/ref/{ref:.+}", Method: "GET", } @@ -2438,12 +2443,12 @@ var PostReposGitRefsByOwnerByRepo EndpointPattern = EndpointPattern{ } var PatchReposGitRefsByOwnerByRepoByRef EndpointPattern = EndpointPattern{ - Pattern: "/repos/{owner}/{repo}/git/refs/{ref}", + Pattern: "/repos/{owner}/{repo}/git/refs/{ref:.+}", Method: "PATCH", } var DeleteReposGitRefsByOwnerByRepoByRef EndpointPattern = EndpointPattern{ - Pattern: "/repos/{owner}/{repo}/git/refs/{ref}", + Pattern: "/repos/{owner}/{repo}/git/refs/{ref:.+}", Method: "DELETE", } From 1cba4e2d2ed51fa20181aea91f42b5e9f3817806 Mon Sep 17 00:00:00 2001 From: Miguel Elias dos Santos Date: Wed, 6 Apr 2022 21:58:27 +1000 Subject: [PATCH 3/3] Add test cases for repo git edge mutators --- src/gen/gen_mutations.go | 2 +- src/mock/endpointpattern_test.go | 66 +++++++++++++++++++++++++++++++- 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/src/gen/gen_mutations.go b/src/gen/gen_mutations.go index 78e8f59..3d9db31 100644 --- a/src/gen/gen_mutations.go +++ b/src/gen/gen_mutations.go @@ -9,7 +9,7 @@ import ( var enabledMutators = map[string]func(ScrapeResult) ScrapeResult{ "/repos/{owner}/{repo}/contents/{path}": allowExtendedLastParamMutatorHelper(), "/repos/{owner}/{repo}/git/ref/{ref}": allowExtendedLastParamMutatorHelper(), - "/repos/{owner}/{repo}/git/refs/{ref}": allowExtendedLastParamMutatorHelper(), //thanks for the consistency, GitHub + "/repos/{owner}/{repo}/git/refs/{ref}": allowExtendedLastParamMutatorHelper(), // thanks for the consistency, GitHub } // allowExtendedLastParamMutatorHelper mutates the last param of the endpoint pattern diff --git a/src/mock/endpointpattern_test.go b/src/mock/endpointpattern_test.go index 60e1173..2e2f536 100644 --- a/src/mock/endpointpattern_test.go +++ b/src/mock/endpointpattern_test.go @@ -7,7 +7,7 @@ import ( "github.com/google/go-github/v41/github" ) -func TestRepoGetContents2(t *testing.T) { +func TestRepoGetContents(t *testing.T) { cases := []struct { name string repositoryContent github.RepositoryContent @@ -70,3 +70,67 @@ func TestRepoGetContents2(t *testing.T) { }(c.repositoryContent)) } } + +func TestPatchGitReference(t *testing.T) { + mockedHTTPClient := NewMockedHTTPClient( + WithRequestMatch( + PatchReposGitRefsByOwnerByRepoByRef, + github.Reference{ + Ref: github.String("refs/heads/new-branch"), + }, + ), + ) + + c := github.NewClient(mockedHTTPClient) + + ctx := context.Background() + + ref, _, _ := c.Git.UpdateRef( + ctx, + "owner", + "repo-name", + &github.Reference{ + Ref: github.String("refs/heads/new-branch"), + Object: &github.GitObject{SHA: github.String("fake-sha")}, + }, + false, + ) + + if *(ref.Ref) != "refs/heads/new-branch" { + t.Errorf( + "ref.Ref is %s, want %s", + *ref.Ref, + "refs/heads/new-branch", + ) + } +} + +func TestGetGitReference(t *testing.T) { + mockedHTTPClient := NewMockedHTTPClient( + WithRequestMatch( + GetReposGitRefByOwnerByRepoByRef, + github.Reference{ + Ref: github.String("refs/heads/new-branch"), + }, + ), + ) + + c := github.NewClient(mockedHTTPClient) + + ctx := context.Background() + + ref, _, _ := c.Git.GetRef( + ctx, + "owner", + "repo-name", + "refs/heads/new-branch", + ) + + if *(ref.Ref) != "refs/heads/new-branch" { + t.Errorf( + "ref.Ref is %s, want %s", + *ref.Ref, + "refs/heads/new-branch", + ) + } +}