diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index f067bb3a5d..b081143818 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -41,14 +41,8 @@ jobs: with: go-version: ${{ steps.go_version.outputs.go_version }} - - name: Install gotip - run: | - go install golang.org/dl/gotip@latest - gotip download - gotip version - - - name: gotip test -fuzz - run: gotip test ./ast -fuzz FuzzParseStatementsAndCompileModules -fuzztime 1h -v -run '^$' + - name: go test -fuzz + run: go test ./ast -fuzz FuzzParseStatementsAndCompileModules -fuzztime 1h -v -run '^$' - name: Dump crashers if: ${{ failure() }} @@ -63,36 +57,6 @@ jobs: status: ${{ job.status }} fields: repo,workflow - fuzzer: - name: Go Fuzzer - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v3 - - - name: Run go-fuzz - run: make ci-go-check-fuzz - - - name: Dump crashers - if: ${{ failure() }} - run: find build/fuzzer/workdir/crashers -name '*.quoted' -print -exec cat {} \; - - - name: Upload Workdir - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: workdir - path: ./build/fuzzer/workdir - - - name: Slack Notification - uses: 8398a7/action-slack@v3 - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFICATION_WEBHOOK }} - if: ${{ failure() && env.SLACK_WEBHOOK_URL }} - with: - status: ${{ job.status }} - fields: repo,workflow - go-proxy-check: name: Go mod check runs-on: ubuntu-latest diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index 415d5fe86b..b9ff66ef5e 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -130,7 +130,7 @@ jobs: uses: actions/checkout@v3 - name: Run fuzz check (3m) - run: make ci-go-check-fuzz FUZZ_TIME=180 + run: make ci-go-fuzz FUZZ_TIME=180s timeout-minutes: 30 go-lint: @@ -280,20 +280,25 @@ jobs: working-directory: npm-opa-wasm go-version-build: - name: Go compat builds - runs-on: ubuntu-latest + name: Go compat build/test + needs: generate + runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: - include: - - version: "1.16" - - version: "1.15" + os: [ubuntu-18.04, macos-latest] + version: ["1.17", "1.16"] steps: - - name: Check out code - uses: actions/checkout@v3 - - - name: Build - run: make ci-go-ci-build-linux GOVERSION=${{ matrix.version }} - timeout-minutes: 30 + - uses: actions/checkout@v3 + - name: Download generated artifacts + uses: actions/download-artifact@v2 + with: + name: generated + - uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.version }} + - run: make build + - run: make go-test # Run PR metadata against Rego policies rego-check-pr: @@ -305,6 +310,8 @@ jobs: - name: Download OPA uses: open-policy-agent/setup-opa@v1 + with: + version: edge - name: Test policies run: opa test build/policy diff --git a/.go-version b/.go-version index b48f322609..adc97d8e22 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.17 +1.18 diff --git a/ADOPTERS.md b/ADOPTERS.md index f6e5ccf84c..350c5b2dea 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -237,6 +237,8 @@ pre-production (in alphabetical order): automated code review, defining access levels or blocking execution of unwanted code. +* [Wealthsimple](https://www.wealthsimple.com/) is using OPA to power all authorization checks their microservice ecosystem by leveraging their existing authorization library make the transition to OPA as simple as possible for development teams. + Other adopters that have gone into production or various stages of testing include: diff --git a/CHANGELOG.md b/CHANGELOG.md index ee27beec4b..85f064fa68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,83 @@ project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased +## 0.39.0 + +This release contains a number of fixes and enhancements. + +### Disk Storage + +The on-disk storage backend has been fully integrated with the OPA server, and +can now be enabled via configuration: + +```yaml +storage: + disk: + directory: /var/opa # put data here + auto_create: true # create directory if it doesn't exist + partitions: # partitioning is important for data storage, + - /users/* # please see the documentation +``` + +It is intended to enable the use of OPA in scenarios where the data needed for +policy evaluation exceeds the available memory. + +The on-disk contents will persist among restarts, but should not be used as a +single source of truth: there are no backup mechanisms, and certain data partitioning +changes will require a start-over. These are things that may get improved in the +future. + +For all the details, please refer to the [configuration](https://www.openpolicyagent.org/docs/v0.39.0/configuration/#disk-storage) +and [detailled Disk Storage section](https://www.openpolicyagent.org/docs/v0.39.0/misc-disk/) +of the documentations. + +### Tooling, SDK, and Runtime + +- Server: Add warning when `input` attribute is missing in `POST /v1/data` API ([#4386](https://github.com/open-policy-agent/opa/issues/4386)) authored by @aflmp +- SDK: Support partial evaluation ([#4240](https://github.com/open-policy-agent/opa/pull/4240)), authored by @kroekle; with a fix to avoid using different state (authored by @Iceber) +- Runtime: Suppress payloads in debug logs for handlers that compress responses (`/metrics` and `/debug/pprof`) (authored by @christian1607) +- `opa test`: Add file path to failing tests to make debugging failing tests easier ([#4457](https://github.com/open-policy-agent/opa/issues/4457)), authored by @liamg +- `opa fmt`: avoid whitespace mixed with tabs on `with` statements ([#4376](https://github.com/open-policy-agent/opa/issues/4376)) reported by @tiwood +- Coverage reporting: Remove duplicates from coverage report ([#4393](https://github.com/open-policy-agent/opa/issues/4393)) reported by @gianna7wu +- Plugins: Fix broken retry logic in decision logs plugin ([#4486](https://github.com/open-policy-agent/opa/issues/4486)) reported by @iamatwork +- Plugins: Update regular polling fallback mechanism for downloader +- Plugins: Support for adding custom parameters and headers for OAuth2 Client Credentials Token request (authored by @srlk) +- Plugins: Log message on unexpected bundle content type ([#4278](https://github.com/open-policy-agent/opa/issues/4278)) +- Plugins: Mask Authorization header value in debug logs ([#4495](https://github.com/open-policy-agent/opa/issues/4495)) +- Docker images: Use GID 1000 in `-rootless` images ([#4380](https://github.com/open-policy-agent/opa/issues/4380)); also warn when using UID/GID 0. +- Runtime: change processed file event log level to info + +### Rego and Topdown + +- Type checker: Skip pattern JSON Schema attribute compilation ([#4426](https://github.com/open-policy-agent/opa/issues/4426)): These are not supported, but could have caused the parsing of a JSON Schema document to fail. +- Topdown: Copy without modifying expr, fixing a bug that could occur when running multiple partial evaluation requests concurrently. +- Compiler strict mode: Raise error on unused imports ([#4354](https://github.com/open-policy-agent/opa/issues/4354)) authored by @damienjburks +- AST: Fix print call rewriting in else rules ([#4489](https://github.com/open-policy-agent/opa/issues/4489)) +- Compiler: Improve error message on missing `with` target ([#4431](https://github.com/open-policy-agent/opa/issues/4431)) reported by @gabrielfern +- Parser: hint about 'every' future keyword import + +### Documentation and Website + +- AWS CloudFormation Hook: New tutorial +- Community: Stretch background so it covers on larger screens ([#4402](https://github.com/open-policy-agent/opa/issues/4402)) authored by @msorens +- Build: Make local dev and PR preview not build everything ([#4379](https://github.com/open-policy-agent/opa/issues/4379)) +- Philosophy: Grammar fixes (authored by @ajonesiii) +- README: Add note about Hugo version mismatch errors (authored by @ogazitt) +- Integrations: Add GraphQL-Graphene (authored by @dolevf), Emissary-Ingress (authored by @tayyabjamadar), rekor-sidekick, +- Integrations CI: ensure referenced software is listed, and logo file names match; allow SVG logos +- Envoy: Update policy primer with new control headers +- Envoy: Update bob_token and alice_token in tutorial (authored by @rokkiter) +- Envoy: Include new configurable gRPC msg sizes (authored by @emaincourt) +- Annotations: add missing title to index (authored by @itaysk) + +### Miscellaneous + +- Various dependency bumps, notably: + - OpenTelemetry-go: 1.4.1 -> 1.6.1 + - Wasmtime-go: 0.34.0 -> 0.35.0 +- Binaries and Docker images are now built using Go 1.18; CI runs build/test for Ubuntu and macos with Go 1.16 and 1.17. +- CI: remove go-fuzz, use native go 1.18 fuzzer + ## 0.38.1 This is a bug fix release that addresses one issue when using `opa test` with the diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 3e7c5ef2e6..7a10306bb0 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -4,15 +4,16 @@ The following table lists OPA project maintainers and areas of expertise in alph | Name | GitHub | Email | Organization | Repositories/Area of Expertise | Added/Renewed On | | --- | --- | --- | --- | --- | --- | -| Ash Narkar | @ashutosh-narkar | anarkar4387@gmail.com | Styra | opa, opa-envoy-plugin | 2021-03-29 | -| Craig Tabita | @ctab | ctab@google.com | Google | gatekeeper, gatekeeper-library, cert-controller | 2021-03-29 | -| Max Smythe | @maxsmythe | smythe@google.com | Google | frameworks/constraints, gatekeeper, gatekeeper-library, cert-controller | 2021-03-29 | +| Ash Narkar | @ashutosh-narkar | anarkar4387@gmail.com | Styra | opa, opa-envoy-plugin | 2022-03-24 | +| Max Smythe | @maxsmythe | smythe@google.com | Google | frameworks/constraints, gatekeeper, gatekeeper-library, cert-controller | 2022-03-24 | | Oren Shomron | @shomron | shomron@gmail.com | VMware | frameworks/constraints, gatekeeper, gatekeeper-library, cert-controller | 2020-11-13 | -| Rita Zhang | @ritazh | rita.z.zhang@gmail.com | Microsoft | frameworks/constraints, gatekeeper, gatekeeper-library, cert-controller | 2021-03-29 | -| Sertaç Özercan | @sozercan | sozercan@gmail.com | Microsoft | gatekeeper, gatekeeper-library, cert-controller | 2021-03-29 | -| Tim Hinrichs | @timothyhinrichs | timothy.l.hinrichs@gmail.com | Styra | all repositories | 2021-03-29 | -| Torin Sandall | @tsandall | torinsandall@gmail.com | Styra | all repositories | 2021-03-29 | +| Rita Zhang | @ritazh | rita.z.zhang@gmail.com | Microsoft | frameworks/constraints, gatekeeper, gatekeeper-library, cert-controller | 2022-03-24 | +| Sertaç Özercan | @sozercan | sozercan@gmail.com | Microsoft | gatekeeper, gatekeeper-library, cert-controller | 2022-03-24 | +| Tim Hinrichs | @timothyhinrichs | timothy.l.hinrichs@gmail.com | Styra | all repositories | 2022-03-24 | +| Torin Sandall | @tsandall | torinsandall@gmail.com | Styra | all repositories | 2022-03-24 | +| Will Beason | @willbeason | willbeason@google.com | Google | gatekeeper, gatekeeper-library, cert-controller | 2022-03-24 | ## Emeritus +* [Craig Tabita](https://github.com/ctab) * [Patrick East](https://github.com/patrick-east) diff --git a/Makefile b/Makefile index b6861d80f7..ad15ed5819 100644 --- a/Makefile +++ b/Makefile @@ -57,7 +57,7 @@ BIN := opa_$(GOOS)_$(GOARCH) # Optional external configuration useful for forks of OPA DOCKER_IMAGE ?= openpolicyagent/opa S3_RELEASE_BUCKET ?= opa-releases -FUZZ_TIME ?= 3600 # 1hr +FUZZ_TIME ?= 1h TELEMETRY_URL ?= #Default empty BUILD_COMMIT := $(shell ./build/get-build-commit.sh) @@ -166,7 +166,8 @@ clean: wasm-lib-clean .PHONY: fuzz fuzz: - $(MAKE) -C ./build/fuzzer all + go test ./ast -fuzz FuzzParseStatementsAndCompileModules -fuzztime ${FUZZ_TIME} -v -run '^$$' + ###################################################### # @@ -447,14 +448,14 @@ release-ci: push-image push-manifest-list-latest endif .PHONY: netlify-prod -netlify-prod: clean docs-clean build docs-generate docs-production-build +netlify-prod: clean docs-clean build docs-production-build .PHONY: netlify-preview -netlify-preview: clean docs-clean build docs-live-blocks-install-deps docs-live-blocks-test docs-generate docs-preview-build +netlify-preview: clean docs-clean build docs-live-blocks-install-deps docs-live-blocks-test docs-dev-generate docs-preview-build +# Kept for compatibility. Use `make fuzz` instead. .PHONY: check-fuzz -check-fuzz: - ./build/check-fuzz.sh $(FUZZ_TIME) +check-fuzz: fuzz # GOPRIVATE=* causes go to fetch all dependencies from their corresponding VCS # source, not through the golang-provided proxy services. We're cleaning out diff --git a/ast/compile.go b/ast/compile.go index e17fa7dfba..6c738d0e10 100644 --- a/ast/compile.go +++ b/ast/compile.go @@ -1496,12 +1496,14 @@ func (c *Compiler) rewritePrintCalls() { WalkRules(mod, func(r *Rule) bool { safe := r.Head.Args.Vars() safe.Update(ReservedVars) - WalkBodies(r, func(b Body) bool { + vis := func(b Body) bool { for _, err := range rewritePrintCalls(c.localvargen, c.GetArity, safe, b) { c.err(err) } return false - }) + } + WalkBodies(r.Head, vis) + WalkBodies(r.Body, vis) return false }) } diff --git a/ast/compile_test.go b/ast/compile_test.go index 03ffac0960..f56c65a28d 100644 --- a/ast/compile_test.go +++ b/ast/compile_test.go @@ -4113,6 +4113,45 @@ func TestCompilerRewritePrintCalls(t *testing.T) { } } +func TestRewritePrintCallsWithElseImplicitArgs(t *testing.T) { + + module := `package test + + f(x, y) { + x = y + } + + else = false { + print(x, y) + }` + + c := NewCompiler().WithEnablePrintStatements(true) + opts := ParserOptions{AllFutureKeywords: true, unreleasedKeywords: true} + c.Compile(map[string]*Module{ + "test.rego": MustParseModuleWithOpts(module, opts), + }) + + if c.Failed() { + t.Fatal(c.Errors) + } + + exp := MustParseModuleWithOpts(`package test + + f(__local0__, __local1__) = true { __local0__ = __local1__ } + else = false { __local6__ = {__local4__ | __local4__ = __local2__}; __local7__ = {__local5__ | __local5__ = __local3__}; internal.print([__local6__, __local7__]) } + `, opts) + + // NOTE(tsandall): we have to patch the implicit args on the else rule + // because of how the parser copies the arg names across from the first + // rule. + exp.Rules[0].Else.Head.Args[0] = VarTerm("__local2__") + exp.Rules[0].Else.Head.Args[1] = VarTerm("__local3__") + + if !exp.Equal(c.Modules["test.rego"]) { + t.Fatalf("Expected:\n\n%v\n\nGot:\n\n%v", exp, c.Modules["test.rego"]) + } +} + func TestCompilerMockFunction(t *testing.T) { c := NewCompiler() c.Modules["test"] = MustParseModule(` @@ -4299,7 +4338,7 @@ func TestCompilerCheckUnusedAssignedVar(t *testing.T) { { note: "rule with nested closure", module: `package test - p { + p { x := 1 a := 1 { y | y := [ z | z:=[1,2,3][a]; z > 1 ][_] } @@ -4312,7 +4351,7 @@ func TestCompilerCheckUnusedAssignedVar(t *testing.T) { { note: "rule with nested closure and unused inner var", module: `package test - p { + p { x := 1 { y | y := [ z | z:=[1,2,3][x]; z > 1; a := 2 ][_] } } diff --git a/ast/policy.go b/ast/policy.go index d1f1c660a0..fdb3d897ed 100644 --- a/ast/policy.go +++ b/ast/policy.go @@ -1053,10 +1053,22 @@ func (expr *Expr) sortOrder() int { return -1 } +// CopyWithoutTerms returns a deep copy of expr without its Terms +func (expr *Expr) CopyWithoutTerms() *Expr { + cpy := *expr + + cpy.With = make([]*With, len(expr.With)) + for i := range expr.With { + cpy.With[i] = expr.With[i].Copy() + } + + return &cpy +} + // Copy returns a deep copy of expr. func (expr *Expr) Copy() *Expr { - cpy := *expr + cpy := expr.CopyWithoutTerms() switch ts := expr.Terms.(type) { case *SomeDecl: @@ -1073,12 +1085,7 @@ func (expr *Expr) Copy() *Expr { cpy.Terms = ts.Copy() } - cpy.With = make([]*With, len(expr.With)) - for i := range expr.With { - cpy.With[i] = expr.With[i].Copy() - } - - return &cpy + return cpy } // Hash returns the hash code of the Expr. diff --git a/build/check-fuzz.sh b/build/check-fuzz.sh deleted file mode 100755 index f4fa5d2a5d..0000000000 --- a/build/check-fuzz.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -set -e - -OPA_DIR=$(dirname "${BASH_SOURCE}")/.. - -usage() { - echo "check-fuzz.sh