diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..d36ed9f22 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,56 @@ +# Golang CircleCI 2.0 configuration file +# +# Check https://circleci.com/docs/2.0/language-go/ for more details +version: 2.1 + +jobs: + "golang-1_15": &template + machine: + # https://circleci.com/docs/2.0/configuration-reference/#available-machine-images + image: ubuntu-2004:202010-01 + # docker_layer_caching: true + + # https://circleci.com/docs/2.0/configuration-reference/#resource_class + resource_class: medium + + # Leave working directory unspecified and use defaults: + # https://circleci.com/blog/go-v1.11-modules-and-circleci/ + # working_directory: /go/src/github.com/golang-migrate/migrate + + environment: + GO111MODULE: "on" + GO_VERSION: "1.15.x" + + steps: + # - setup_remote_docker: + # version: 19.03.13 + # docker_layer_caching: true + - run: curl -sL -o ~/bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme + - run: curl -sfL -o ~/bin/golangci-lint.sh https://install.goreleaser.com/github.com/golangci/golangci-lint.sh + - run: chmod +x ~/bin/gimme ~/bin/golangci-lint.sh + - run: eval "$(gimme $GO_VERSION)" + - run: golangci-lint.sh -b ~/bin v1.37.0 + - checkout + - restore_cache: + keys: + - go-mod-v1-{{ arch }}-{{ checksum "go.sum" }} + - run: golangci-lint run + - run: make test COVERAGE_DIR=/tmp/coverage + - save_cache: + key: go-mod-v1-{{ arch }}-{{ checksum "go.sum" }} + paths: + - "/go/pkg/mod" + - run: go get github.com/mattn/goveralls + - run: goveralls -service=circle-ci -coverprofile /tmp/coverage/combined.txt + + "golang-1_16": + <<: *template + environment: + GO_VERSION: "1.16.x" + +workflows: + version: 2 + build: + jobs: + - "golang-1_15" + - "golang-1_16" diff --git a/.dockerignore b/.dockerignore index df33687f9..f12dc01d3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,7 +2,6 @@ FAQ.md README.md LICENSE -Makefile .gitignore .travis.yml CONTRIBUTING.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 6b085e03d..d63378f1a 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -25,7 +25,7 @@ e.g. s3, github, go-bindata, gcs, file Obtained by running: `migrate -help` **Loaded Database Drivers** -e.g. spanner, stub, clickhouse, cockroachdb, crdb-postgres, postgres, postgresql, redshift, cassandra, cockroach, mysql +e.g. spanner, stub, clickhouse, cockroachdb, crdb-postgres, postgres, postgresql, pgx, redshift, cassandra, cockroach, mysql Obtained by running: `migrate -help` **Go Version** diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 000000000..61842f7ba --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,97 @@ +name: CI + +on: + push: + pull_request: + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: golangci-lint + uses: golangci/golangci-lint-action@v2 + + test: + runs-on: ubuntu-latest + strategy: + matrix: + go: ["1.16.x", "1.17.x"] + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} + + - name: Run test + run: make test COVERAGE_DIR=/tmp/coverage + + - name: Send goveralls coverage + uses: shogo82148/actions-goveralls@v1 + with: + path-to-profile: /tmp/coverage/combined.txt + flag-name: Go-${{ matrix.go }} + parallel: true + + check-coverage: + name: Check coverage + needs: [test] + runs-on: ubuntu-latest + steps: + - uses: shogo82148/actions-goveralls@v1 + with: + parallel-finished: true + + goreleaser: + name: Release a new version + needs: [lint, test] + runs-on: ubuntu-latest + environment: GoReleaser + # This job only runs when + # 1. When the previous `lint` and `test` jobs has completed successfully + # 2. When the repository is not a fork, i.e. it will only run on the official golang-migrate/migrate + # 3. When the workflow is triggered by a tag with `v` prefix + if: ${{ success() && github.repository == 'golang-migrate/migrate' && startsWith(github.ref, 'refs/tags/v') }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: 2.7 + - uses: actions/setup-go@v2 + with: + go-version: "1.17.x" + + - uses: docker/setup-qemu-action@v1 + - uses: docker/setup-buildx-action@v1 + - uses: docker/login-action@v1 + with: + username: golangmigrate + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - run: echo "SOURCE=$(make echo-source)" >> $GITHUB_ENV + - run: echo "DATABASE=$(make echo-database)" >> $GITHUB_ENV + + - uses: goreleaser/goreleaser-action@v2 + with: + version: latest + args: release --rm-dist + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - run: gem install package_cloud + - run: package_cloud push golang-migrate/migrate/ubuntu/bionic dist/migrate.linux-amd64.deb + env: + PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} + - run: package_cloud push golang-migrate/migrate/ubuntu/focal dist/migrate.linux-amd64.deb + env: + PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} + - run: package_cloud push golang-migrate/migrate/debian/buster dist/migrate.linux-amd64.deb + env: + PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} + - run: package_cloud push golang-migrate/migrate/debian/bullseye dist/migrate.linux-amd64.deb + env: + PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} diff --git a/.gitignore b/.gitignore index a8eaf9bf7..23b56045c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -.idea .DS_Store cli/build cli/cli @@ -6,3 +5,6 @@ cli/migrate .coverage .godoc.pid vendor/ +.vscode/ +.idea +dist/ diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 000000000..0419e32be --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,26 @@ +run: + # timeout for analysis, e.g. 30s, 5m, default is 1m + timeout: 5m +linters: + enable: + #- golint + - interfacer + - unconvert + #- dupl + - goconst + - gofmt + - misspell + - unparam + - nakedret + - prealloc + #- gosec +linters-settings: + misspell: + locale: US +issues: + max-same-issues: 0 + max-issues-per-linter: 0 + exclude-use-default: false + exclude: + # gosec: Duplicated errcheck checks + - G104 diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 000000000..682248f65 --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,102 @@ +project_name: migrate +before: + hooks: + - go mod tidy +builds: + - env: + - CGO_ENABLED=0 + goos: + - linux + - windows + - darwin + goarch: + - amd64 + - arm + - arm64 + - 386 + goarm: + - 7 + main: ./cmd/migrate + ldflags: + - '-w -s -X main.Version={{ .Version }} -extldflags "static"' + flags: + - "-tags={{ .Env.DATABASE }} {{ .Env.SOURCE }}" + - "-trimpath" +nfpms: + - homepage: "https://github.com/golang-migrate/migrate" + maintainer: "dhui@users.noreply.github.com" + license: MIT + description: "Database migrations" + formats: + - deb + file_name_template: "{{ .ProjectName }}.{{ .Os }}-{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}" +dockers: + - goos: linux + goarch: amd64 + dockerfile: Dockerfile.github-actions + use: buildx + ids: + - migrate + image_templates: + - 'migrate/migrate:{{ .Tag }}-amd64' + build_flag_templates: + - '--label=org.opencontainers.image.created={{ .Date }}' + - '--label=org.opencontainers.image.title={{ .ProjectName }}' + - '--label=org.opencontainers.image.revision={{ .FullCommit }}' + - '--label=org.opencontainers.image.version={{ .Version }}' + - "--label=org.opencontainers.image.source={{ .GitURL }}" + - "--platform=linux/amd64" + - goos: linux + goarch: arm64 + dockerfile: Dockerfile.github-actions + use: buildx + ids: + - migrate + image_templates: + - 'migrate/migrate:{{ .Tag }}-arm64' + build_flag_templates: + - '--label=org.opencontainers.image.created={{ .Date }}' + - '--label=org.opencontainers.image.title={{ .ProjectName }}' + - '--label=org.opencontainers.image.revision={{ .FullCommit }}' + - '--label=org.opencontainers.image.version={{ .Version }}' + - "--label=org.opencontainers.image.source={{ .GitURL }}" + - "--platform=linux/arm64" + +docker_manifests: +- name_template: 'migrate/migrate:{{ .Tag }}' + image_templates: + - 'migrate/migrate:{{ .Tag }}-amd64' + - 'migrate/migrate:{{ .Tag }}-arm64' +- name_template: 'migrate/migrate:{{ .Major }}' + image_templates: + - 'migrate/migrate:{{ .Tag }}-amd64' + - 'migrate/migrate:{{ .Tag }}-arm64' +- name_template: 'migrate/migrate:latest' + image_templates: + - 'migrate/migrate:{{ .Tag }}-amd64' + - 'migrate/migrate:{{ .Tag }}-arm64' +archives: + - name_template: "{{ .ProjectName }}.{{ .Os }}-{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}" + format_overrides: + - goos: windows + format: zip +checksum: + name_template: 'sha256sum.txt' +release: + draft: true + prerelease: auto +source: + enabled: true + format: zip +changelog: + skip: false + sort: asc + filters: + exclude: + - '^docs:' + - '^test:' + - Merge pull request + - Merge branch + - go mod tidy +snapshot: + name_template: "{{ .Tag }}-next" diff --git a/.travis.yml b/.travis.yml index 9e2890635..fdaea8cbd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,9 +6,8 @@ matrix: - go: master include: # Supported versions of Go: https://golang.org/dl/ - - go: "1.10.x" - - go: "1.11.x" - - go: "1.13.x" + - go: "1.14.x" + - go: "1.15.x" - go: master go_import_path: github.com/golang-migrate/migrate @@ -23,37 +22,34 @@ env: services: - docker -before_cache: - - mv $GOPATH/src/github.com/golang-migrate /tmp/golang-migrate - - rm -rf $GOPATH/pkg/**/github.com/golang-migrate - cache: directories: - $GOPATH/pkg + before_install: + # Update docker to latest version: https://docs.travis-ci.com/user/docker/#installing-a-newer-docker-version + - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + - sudo apt-get update + - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce + # Install golangci-lint + - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.30.0 - echo "TRAVIS_GO_VERSION=${TRAVIS_GO_VERSION}" - # Download the binary to bin folder in $GOPATH - - if [[ "${TRAVIS_GO_VERSION}" == 1.10* ]]; then curl -L -s https://github.com/golang/dep/releases/download/v0.5.0/dep-linux-amd64 -o $GOPATH/bin/dep; fi - # Make the binary executable - - if [[ "${TRAVIS_GO_VERSION}" == 1.10* ]]; then chmod +x $GOPATH/bin/dep; fi install: - - if [[ "${TRAVIS_GO_VERSION}" == 1.10* ]]; then dep ensure -vendor-only; fi - go get github.com/mattn/goveralls script: + - golangci-lint run - make test COVERAGE_DIR=/tmp/coverage after_success: - - mv /tmp/golang-migrate $GOPATH/src/github.com/golang-migrate - goveralls -service=travis-ci -coverprofile /tmp/coverage/combined.txt - make list-external-deps > dependency_tree.txt && cat dependency_tree.txt - -before_deploy: - make build-cli - - gem install --no-ri --no-rdoc fpm - - fpm -s dir -t deb -n migrate -v "$(git describe --tags 2>/dev/null | cut -c 2-)" --license MIT -m dhui@users.noreply.github.com --url https://github.com/golang-migrate/migrate --description='Database migrations' -a amd64 -p migrate.$(git describe --tags 2>/dev/null | cut -c 2-).deb --deb-no-default-config-files -f -C cli/build migrate.linux-amd64=/usr/bin/migrate + - gem install --no-document fpm + - fpm -s dir -t deb -n migrate -v "$(git describe --tags 2>/dev/null | cut -c 2-)" --license MIT -m dhui@users.noreply.github.com --url https://github.com/golang-migrate/migrate --description='Database migrations' -a amd64 -p migrate.$(git describe --tags 2>/dev/null | cut -c 2-).deb --deb-no-default-config-files -f -C cli/build migrate.linux-amd64=/usr/local/bin/migrate deploy: - provider: releases @@ -61,13 +57,16 @@ deploy: secure: hWH1HLPpzpfA8pXQ93T1qKQVFSpQp0as/JLQ7D91jHuJ8p+RxVeqblDrR6HQY/95R/nyiE9GJmvUolSuw5h449LSrGxPtVWhdh6EnkxlQHlen5XeMhVjRjFV0sE9qGe8v7uAkiTfRO61ktTWHrEAvw5qpyqnNISodmZS78XIasPODQbNlzwINhWhDTHIjXGb4FpizYaL3OGCanrxfR9fQyCaqKGGBjRq3Mfq8U6Yd4mApmsE+uJxgaZV8K5zBqpkSzQRWhcVGNL5DuLsU3gfSJOo7kZeA2G71SHffH577dBoqtCZ4VFv169CoUZehLWCb+7XKJZmHXVujCURATSySLGUOPc6EoLFAn3YtsCA04mS4bZVo5FZPWVwfhjmkhtDR4f6wscKp7r1HsFHSOgm59QfETQdrn4MnZ44H2Jd39axqndn5DvK9EcZVjPHynOPnueXP2u6mTuUgh2VyyWBCDO3CNo0fGlo7VJI69IkIWNSD87K9cHZWYMClyKZkUzS+PmRAhHRYbVd+9ZjKOmnU36kUHNDG/ft1D4ogsY+rhVtXB4lgWDM5adri+EIScYdYnB1/pQexLBigcJY9uE7nQTR0U6QgVNYvun7uRNs40E0c4voSfmPdFO0FlOD2y1oQhnaXfWLbu9nMcTcs4RFGrcC7NzkUN4/WjG8s285V6w= skip_cleanup: true on: - go: "1.13.x" + go: "1.15.x" repo: golang-migrate/migrate tags: true file: - cli/build/migrate.linux-amd64.tar.gz + - cli/build/migrate.linux-armv7.tar.gz + - cli/build/migrate.linux-arm64.tar.gz - cli/build/migrate.darwin-amd64.tar.gz - cli/build/migrate.windows-amd64.exe.tar.gz + - cli/build/migrate.windows-386.exe.tar.gz - cli/build/sha256sum.txt - dependency_tree.txt - provider: packagecloud @@ -79,12 +78,61 @@ deploy: package_glob: '*.deb' skip_cleanup: true on: - go: "1.13.x" + go: "1.15.x" + repo: golang-migrate/migrate + tags: true + - provider: packagecloud + repository: migrate + username: golang-migrate + token: + secure: aICwu3gJ1sJ1QVCD3elpg+Jxzt4P+Zj1uoh5f0sOwnjDNIZ4FwUT1cMrWloP8P2KD0iyCOawuZER27o/kQ21oX2OxHvQbYPReA2znLm7lHzCmypAAOHPxpgnQ4rMGHHJXd+OsxtdclGs67c+EbdBfoRRbK400Qz/vjPJEDeH4mh02ZHC2nw4Nk/wV4jjBIkIt9dGEx6NgOA17FCMa3MaPHlHeFIzU7IfTlDHbS0mCCYbg/wafWBWcbGqtZLWAYtJDmfjrAStmDLdAX5J5PsB7taGSGPZHmPmpGoVgrKt/tb9Xz1rFBGslTpGROOiO4CiMAvkEKFn8mxrBGjfSBqp7Dp3eeSalKXB1DJAbEXx2sEbMcvmnoR9o43meaAn+ZRts8lRL8S/skBloe6Nk8bx3NlJCGB9WPK1G56b7c/fZnJxQbrCw6hxDfbZwm8S2YPviFTo/z1BfZDhRsL74reKsN2kgnGo2W/k38vvzIpsssQ9DHN1b0TLCxolCNPtQ7oHcQ1ohcjP2UgYXk0FhqDoL+9LQva/DU4N9sKH0UbAaqsMVSErLeG8A4aauuFcVrWRBaDYyTag4dQqzTulEy7iru2kDDIBgSQ1gMW/yoBOIPK4oi6MtbTf1X39fzXFLS1cDd3LW61yAu3YrbjAetpfx2frIvrRAiL9TxWA1gnrs5o= + dist: ubuntu/bionic + package_glob: '*.deb' + skip_cleanup: true + on: + go: "1.15.x" + repo: golang-migrate/migrate + tags: true + - provider: packagecloud + repository: migrate + username: golang-migrate + token: + secure: aICwu3gJ1sJ1QVCD3elpg+Jxzt4P+Zj1uoh5f0sOwnjDNIZ4FwUT1cMrWloP8P2KD0iyCOawuZER27o/kQ21oX2OxHvQbYPReA2znLm7lHzCmypAAOHPxpgnQ4rMGHHJXd+OsxtdclGs67c+EbdBfoRRbK400Qz/vjPJEDeH4mh02ZHC2nw4Nk/wV4jjBIkIt9dGEx6NgOA17FCMa3MaPHlHeFIzU7IfTlDHbS0mCCYbg/wafWBWcbGqtZLWAYtJDmfjrAStmDLdAX5J5PsB7taGSGPZHmPmpGoVgrKt/tb9Xz1rFBGslTpGROOiO4CiMAvkEKFn8mxrBGjfSBqp7Dp3eeSalKXB1DJAbEXx2sEbMcvmnoR9o43meaAn+ZRts8lRL8S/skBloe6Nk8bx3NlJCGB9WPK1G56b7c/fZnJxQbrCw6hxDfbZwm8S2YPviFTo/z1BfZDhRsL74reKsN2kgnGo2W/k38vvzIpsssQ9DHN1b0TLCxolCNPtQ7oHcQ1ohcjP2UgYXk0FhqDoL+9LQva/DU4N9sKH0UbAaqsMVSErLeG8A4aauuFcVrWRBaDYyTag4dQqzTulEy7iru2kDDIBgSQ1gMW/yoBOIPK4oi6MtbTf1X39fzXFLS1cDd3LW61yAu3YrbjAetpfx2frIvrRAiL9TxWA1gnrs5o= + dist: ubuntu/focal + package_glob: '*.deb' + skip_cleanup: true + on: + go: "1.15.x" + repo: golang-migrate/migrate + tags: true + - provider: packagecloud + repository: migrate + username: golang-migrate + token: + secure: aICwu3gJ1sJ1QVCD3elpg+Jxzt4P+Zj1uoh5f0sOwnjDNIZ4FwUT1cMrWloP8P2KD0iyCOawuZER27o/kQ21oX2OxHvQbYPReA2znLm7lHzCmypAAOHPxpgnQ4rMGHHJXd+OsxtdclGs67c+EbdBfoRRbK400Qz/vjPJEDeH4mh02ZHC2nw4Nk/wV4jjBIkIt9dGEx6NgOA17FCMa3MaPHlHeFIzU7IfTlDHbS0mCCYbg/wafWBWcbGqtZLWAYtJDmfjrAStmDLdAX5J5PsB7taGSGPZHmPmpGoVgrKt/tb9Xz1rFBGslTpGROOiO4CiMAvkEKFn8mxrBGjfSBqp7Dp3eeSalKXB1DJAbEXx2sEbMcvmnoR9o43meaAn+ZRts8lRL8S/skBloe6Nk8bx3NlJCGB9WPK1G56b7c/fZnJxQbrCw6hxDfbZwm8S2YPviFTo/z1BfZDhRsL74reKsN2kgnGo2W/k38vvzIpsssQ9DHN1b0TLCxolCNPtQ7oHcQ1ohcjP2UgYXk0FhqDoL+9LQva/DU4N9sKH0UbAaqsMVSErLeG8A4aauuFcVrWRBaDYyTag4dQqzTulEy7iru2kDDIBgSQ1gMW/yoBOIPK4oi6MtbTf1X39fzXFLS1cDd3LW61yAu3YrbjAetpfx2frIvrRAiL9TxWA1gnrs5o= + dist: debian/stretch + package_glob: '*.deb' + skip_cleanup: true + on: + go: "1.15.x" + repo: golang-migrate/migrate + tags: true + - provider: packagecloud + repository: migrate + username: golang-migrate + token: + secure: aICwu3gJ1sJ1QVCD3elpg+Jxzt4P+Zj1uoh5f0sOwnjDNIZ4FwUT1cMrWloP8P2KD0iyCOawuZER27o/kQ21oX2OxHvQbYPReA2znLm7lHzCmypAAOHPxpgnQ4rMGHHJXd+OsxtdclGs67c+EbdBfoRRbK400Qz/vjPJEDeH4mh02ZHC2nw4Nk/wV4jjBIkIt9dGEx6NgOA17FCMa3MaPHlHeFIzU7IfTlDHbS0mCCYbg/wafWBWcbGqtZLWAYtJDmfjrAStmDLdAX5J5PsB7taGSGPZHmPmpGoVgrKt/tb9Xz1rFBGslTpGROOiO4CiMAvkEKFn8mxrBGjfSBqp7Dp3eeSalKXB1DJAbEXx2sEbMcvmnoR9o43meaAn+ZRts8lRL8S/skBloe6Nk8bx3NlJCGB9WPK1G56b7c/fZnJxQbrCw6hxDfbZwm8S2YPviFTo/z1BfZDhRsL74reKsN2kgnGo2W/k38vvzIpsssQ9DHN1b0TLCxolCNPtQ7oHcQ1ohcjP2UgYXk0FhqDoL+9LQva/DU4N9sKH0UbAaqsMVSErLeG8A4aauuFcVrWRBaDYyTag4dQqzTulEy7iru2kDDIBgSQ1gMW/yoBOIPK4oi6MtbTf1X39fzXFLS1cDd3LW61yAu3YrbjAetpfx2frIvrRAiL9TxWA1gnrs5o= + dist: debian/buster + package_glob: '*.deb' + skip_cleanup: true + on: + go: "1.15.x" repo: golang-migrate/migrate tags: true - provider: script script: ./docker-deploy.sh + skip_cleanup: true on: - go: "1.13.x" + go: "1.15.x" repo: golang-migrate/migrate tags: true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 637c0190a..84fb8238a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,8 +2,11 @@ 1. Make sure you have a running Docker daemon (Install for [MacOS](https://docs.docker.com/docker-for-mac/)) + 1. Use a version of Go that supports [modules](https://golang.org/cmd/go/#hdr-Modules__module_versions__and_more) (e.g. Go 1.11+) 1. Fork this repo and `git clone` somewhere to `$GOPATH/src/github.com/golang-migrate/migrate` - 1. Install [dep](https://github.com/golang/dep) and run `dep ensure` to pull dependencies + * Ensure that [Go modules are enabled](https://golang.org/cmd/go/#hdr-Preliminary_module_support) (e.g. your repo path or the `GO111MODULE` environment variable are set correctly) + 1. Install [golangci-lint](https://github.com/golangci/golangci-lint#install) + 1. Run the linter: `golangci-lint run` 1. Confirm tests are working: `make test-short` 1. Write awesome code ... 1. `make test` to run all tests against all database versions diff --git a/Dockerfile b/Dockerfile index af06f70a5..b331066cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,24 +1,25 @@ -FROM golang:1.13-alpine3.12 AS downloader +FROM golang:1.16-alpine3.13 AS builder ARG VERSION -RUN apk add --no-cache git gcc musl-dev +RUN apk add --no-cache git gcc musl-dev make WORKDIR /go/src/github.com/infobloxopen/migrate -COPY . ./ - ENV GO111MODULE=on -ENV DATABASES="postgres mysql redshift cassandra spanner cockroachdb clickhouse" -ENV SOURCES="file go_bindata github aws_s3 google_cloud_storage" -RUN go build -a -o build/migrate.linux-386 -ldflags="-X main.Version=${VERSION}" -tags "$DATABASES $SOURCES" ./cli +COPY go.mod go.sum ./ + +RUN go mod download + +COPY . ./ -FROM alpine:3.12 +RUN make build-docker -RUN apk add --no-cache ca-certificates +FROM alpine:3.13 -COPY --from=downloader /go/src/github.com/infobloxopen/migrate/cli/config /cli/config/ -COPY --from=downloader /go/src/github.com/infobloxopen/migrate/build/migrate.linux-386 /migrate +COPY --from=builder /go/src/github.com/infobloxopen/migrate/cmd/migrate/config /cli/config/ +COPY --from=builder /go/src/github.com/infobloxopen/migrate/build/migrate.linux-386 /usr/local/bin/migrate +RUN ln -s /usr/local/bin/migrate /migrate -ENTRYPOINT ["/migrate"] +ENTRYPOINT ["migrate"] CMD ["--help"] diff --git a/Dockerfile.circleci b/Dockerfile.circleci new file mode 100644 index 000000000..b6b244d19 --- /dev/null +++ b/Dockerfile.circleci @@ -0,0 +1,17 @@ +ARG DOCKER_IMAGE +FROM $DOCKER_IMAGE + +RUN apk add --no-cache git gcc musl-dev make + +WORKDIR /go/src/github.com/golang-migrate/migrate + +ENV GO111MODULE=on +ENV COVERAGE_DIR=/tmp/coverage + +COPY go.mod go.sum ./ + +RUN go mod download + +COPY . ./ + +CMD ["make", "test"] diff --git a/Dockerfile.github-actions b/Dockerfile.github-actions new file mode 100644 index 000000000..559c1e79c --- /dev/null +++ b/Dockerfile.github-actions @@ -0,0 +1,11 @@ +FROM alpine:3.13 + +RUN apk add --no-cache ca-certificates + +COPY migrate /usr/local/bin/migrate + +RUN ln -s /usr/local/bin/migrate /usr/bin/migrate +RUN ln -s /usr/local/bin/migrate /migrate + +ENTRYPOINT ["migrate"] +CMD ["--help"] \ No newline at end of file diff --git a/FAQ.md b/FAQ.md index f8bb9a85b..283162819 100644 --- a/FAQ.md +++ b/FAQ.md @@ -16,7 +16,7 @@ NilMigration defines a migration without a body. NilVersion is defined as const -1. #### What is the difference between uint(version) and int(targetVersion)? - version refers to an existing migration version coming from a source and therefor can never be negative. + version refers to an existing migration version coming from a source and therefore can never be negative. targetVersion can either be a version OR represent a NilVersion, which equals -1. #### What's the difference between Next/Previous and Up/Down? @@ -53,7 +53,7 @@ Yes, technically thats possible. We want to encourage you to contribute your driver to this respository though. The driver's functionality is dictated by migrate's interfaces. That means there should really just be one driver for a database/ source. We want to prevent a future where several drivers doing the exact same thing, - just implemented a bit differently, co-exist somewhere on Github. If users have to do research first to find the + just implemented a bit differently, co-exist somewhere on GitHub. If users have to do research first to find the "best" available driver for a database in order to get started, we would have failed as an open source community. #### Can I mix multiple sources during a batch of migrations? @@ -64,4 +64,16 @@ which prevents attempts to run more migrations on top of a failed migration. You need to manually fix the error and then "force" the expected version. +#### What happens if two programs try and update the database at the same time? +Database-specific locking features are used by *some* database drivers to prevent multiple instances of migrate from running migrations at the same time + the same database at the same time. For example, the MySQL driver uses the `GET_LOCK` function, while the Postgres driver uses + the `pg_advisory_lock` function. +#### Do I need to create a table for tracking migration version used? +No, it is done automatically. + +#### Can I use migrate with a non-Go project? +Yes, you can use the migrate CLI in a non-Go project, but there are probably other libraries/frameworks available that offer better test and deploy integrations in that language/framework. + +#### I have got an error `Dirty database version 1. Fix and force version`. What should I do? +Keep calm and refer to [the getting started docs](GETTING_STARTED.md#forcing-your-database-version). diff --git a/GETTING_STARTED.md b/GETTING_STARTED.md new file mode 100644 index 000000000..5946005cd --- /dev/null +++ b/GETTING_STARTED.md @@ -0,0 +1,53 @@ +# Getting started +Before you start, you should understand the concept of forward/up and reverse/down database migrations. + +Configure a database for your application. Make sure that your database driver is supported [here](README.md#databases) + +## Create migrations +Create some migrations using migrate CLI. Here is an example: +``` +migrate create -ext sql -dir db/migrations -seq create_users_table +``` +Once you create your files, you should fill them. + +**IMPORTANT:** In a project developed by more than one person there is a chance of migrations inconsistency - e.g. two developers can create conflicting migrations, and the developer that created his migration later gets it merged to the repository first. +Developers and Teams should keep an eye on such cases (especially during code review). +[Here](https://github.com/golang-migrate/migrate/issues/179#issuecomment-475821264) is the issue summary if you would like to read more. + +Consider making your migrations idempotent - we can run the same sql code twice in a row with the same result. This makes our migrations more robust. On the other hand, it causes slightly less control over database schema - e.g. let's say you forgot to drop the table in down migration. You run down migration - the table is still there. When you run up migration again - `CREATE TABLE` would return an error, helping you find an issue in down migration, while `CREATE TABLE IF NOT EXISTS` would not. Use those conditions wisely. + +In case you would like to run several commands/queries in one migration, you should wrap them in a transaction (if your database supports it). +This way if one of commands fails, our database will remain unchanged. + +## Run migrations +Run your migrations through the CLI or your app and check if they applied expected changes. +Just to give you an idea: +``` +migrate -database YOUR_DATABASE_URL -path PATH_TO_YOUR_MIGRATIONS up +``` + +Just add the code to your app and you're ready to go! + +Before commiting your migrations you should run your migrations up, down, and then up again to see if migrations are working properly both ways. +(e.g. if you created a table in a migration but reverse migration did not delete it, you will encounter an error when running the forward migration again) +It's also worth checking your migrations in a separate, containerized environment. You can find some tools in the end of this document. + +**IMPORTANT:** If you would like to run multiple instances of your app on different machines be sure to use a database that supports locking when running migrations. Otherwise you may encounter issues. + +## Forcing your database version +In case you run a migration that contained an error, migrate will not let you run other migrations on the same database. You will see an error like `Dirty database version 1. Fix and force version`, even when you fix the erred migration. This means your database was marked as 'dirty'. +You need to investigate the migration error - was your migration applied partially, or was it not applied at all? Once you know, you should force your database to a version reflecting it's real state. You can do so with `force` command: +``` +migrate -path PATH_TO_YOUR_MIGRATIONS -database YOUR_DATABASE_URL force VERSION +``` +Once you force the version and your migration was fixed, your database is 'clean' again and you can proceed with your migrations. + +For details and example of usage see [this comment](https://github.com/golang-migrate/migrate/issues/282#issuecomment-530743258). + +## Further reading: +- [PostgreSQL tutorial](database/postgres/TUTORIAL.md) +- [Best practices](MIGRATIONS.md) +- [FAQ](FAQ.md) +- Tools for testing your migrations in a container: + - https://github.com/dhui/dktest + - https://github.com/ory/dockertest diff --git a/Gopkg.lock b/Gopkg.lock deleted file mode 100644 index 0804902e9..000000000 --- a/Gopkg.lock +++ /dev/null @@ -1,717 +0,0 @@ -# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. - - -[[projects]] - digest = "1:098b27970336695d686ef8a701328cfb231445e2d54564d09c3b0c7d43ce129a" - name = "cloud.google.com/go" - packages = [ - "civil", - "compute/metadata", - "iam", - "internal", - "internal/atomiccache", - "internal/fields", - "internal/optional", - "internal/protostruct", - "internal/trace", - "internal/version", - "longrunning", - "longrunning/autogen", - "spanner", - "spanner/admin/database/apiv1", - "storage", - ] - pruneopts = "UT" - revision = "0fd7230b2a7505833d5f69b75cbd6c9582401479" - version = "v0.23.0" - -[[projects]] - digest = "1:adbfc2db750d6fbe0c165ad801ba2d730883a3bc40bc8fe829516eb42ed17771" - name = "github.com/Microsoft/go-winio" - packages = ["."] - pruneopts = "UT" - revision = "a6d595ae73cf27a1b8fc32930668708f45ce1c85" - version = "v0.4.9" - -[[projects]] - branch = "master" - digest = "1:e401263ad228a4761a67c1de1438187c769c7bd4733067e9642816e303ba4c2f" - name = "github.com/Sirupsen/logrus" - packages = ["."] - pruneopts = "UT" - revision = "f3df9aeffda7c12bd9f5a03f9251d75d35993165" - -[[projects]] - digest = "1:1a42d6d0e8fe8cc1024ed542c5f8a0bc7346a292701b3bd94a07996f29fcbd38" - name = "github.com/aws/aws-sdk-go" - packages = [ - "aws", - "aws/awserr", - "aws/awsutil", - "aws/client", - "aws/client/metadata", - "aws/corehandlers", - "aws/credentials", - "aws/credentials/ec2rolecreds", - "aws/credentials/endpointcreds", - "aws/credentials/stscreds", - "aws/csm", - "aws/defaults", - "aws/ec2metadata", - "aws/endpoints", - "aws/request", - "aws/session", - "aws/signer/v4", - "internal/sdkio", - "internal/sdkrand", - "internal/sdkuri", - "internal/shareddefaults", - "private/protocol", - "private/protocol/eventstream", - "private/protocol/eventstream/eventstreamapi", - "private/protocol/query", - "private/protocol/query/queryutil", - "private/protocol/rest", - "private/protocol/restxml", - "private/protocol/xml/xmlutil", - "service/s3", - "service/s3/s3iface", - "service/sts", - ] - pruneopts = "UT" - revision = "468b9714c11f10b22e533253b35eb9c28f4be691" - version = "v1.14.32" - -[[projects]] - branch = "master" - digest = "1:568184e644ca0114e16fa472037e18bb23a8c0668f9da12f3d2b059e0c548637" - name = "github.com/cockroachdb/cockroach-go" - packages = ["crdb"] - pruneopts = "UT" - revision = "59c0560478b705bf9bd12f9252224a0fad7c87df" - -[[projects]] - branch = "master" - digest = "1:3ba3d94d45f2d5c4b3411452474d96996b7e0dce730b689920f7c307511e5655" - name = "github.com/cznic/b" - packages = ["."] - pruneopts = "UT" - revision = "35e9bbe41f07452a183c517a5fc5f3c9f45eaa0f" - -[[projects]] - branch = "master" - digest = "1:4688e1d4b22367c5bd24fe65f44b3e53a443a93fd19cf34169f1c8337d93dcf7" - name = "github.com/cznic/fileutil" - packages = ["."] - pruneopts = "UT" - revision = "6a051e75936f623600b67c2b1116b6b6c0ffb936" - -[[projects]] - digest = "1:a9e2943ed681d4758a43d1bd9e80d7e993b66ce5adc7aa5dcc083ba48a47a4eb" - name = "github.com/cznic/internal" - packages = [ - "buffer", - "file", - "slice", - ] - pruneopts = "UT" - revision = "cef02a853c3a93623c42eacd574e7ea05f55531b" - version = "1.0.0" - -[[projects]] - digest = "1:ec7b58207ac8eee1c554b7947b3f5124f711238a80edb7224a53bae7263c95ae" - name = "github.com/cznic/lldb" - packages = ["."] - pruneopts = "UT" - revision = "bea8611dd5c407f3c5eab9f9c68e887a27dc6f0e" - version = "v1.1.0" - -[[projects]] - branch = "master" - digest = "1:42951d361293c2f7068ab0411afeb57856afba843fe681cea1e2731b47021222" - name = "github.com/cznic/mathutil" - packages = ["."] - pruneopts = "UT" - revision = "ca4c9f2c136954238c3158b92de72078c7672ecc" - -[[projects]] - digest = "1:905947603b7f1066a7149576b2cbad2e6f890729f892c827cdbcee046d07db9c" - name = "github.com/cznic/ql" - packages = [ - ".", - "driver", - "vendored/github.com/camlistore/go4/lock", - ] - pruneopts = "UT" - revision = "7a63cd7aa46ecd9f549b96983029576af2178f60" - version = "v1.1.0" - -[[projects]] - branch = "master" - digest = "1:06877d57543857a87186f43b6195bbdf5e8f4a0f51412fdf4baef504cead7973" - name = "github.com/cznic/sortutil" - packages = ["."] - pruneopts = "UT" - revision = "4c7342852e65c2088c981288f2c5610d10b9f7f4" - -[[projects]] - branch = "master" - digest = "1:8a26e11e94f1c9f03ab9fc1e816b003629d41b26ab41eefcb7444fad5b24fc4b" - name = "github.com/cznic/strutil" - packages = ["."] - pruneopts = "UT" - revision = "529a34b1c186b483642a7a230c67521d9aa4b0fb" - -[[projects]] - branch = "master" - digest = "1:bb8236408f88f411f565dffcf1e648fe34c7456820b707f96ccbfb89e1cd8eeb" - name = "github.com/cznic/zappy" - packages = ["."] - pruneopts = "UT" - revision = "2533cb5b45cc6c07421468ce262899ddc9d53fb7" - -[[projects]] - branch = "master" - digest = "1:4189ee6a3844f555124d9d2656fe7af02fca961c2a9bad9074789df13a0c62e0" - name = "github.com/docker/distribution" - packages = [ - "digestset", - "reference", - ] - pruneopts = "UT" - revision = "0dae0957e5fe3156c265d22bef4cba9efbf388e2" - -[[projects]] - digest = "1:ec821dda59d7dd340498d74f798aa218b2c782bba54a690e866dc4f520d900d5" - name = "github.com/docker/docker" - packages = [ - "api", - "api/types", - "api/types/blkiodev", - "api/types/container", - "api/types/events", - "api/types/filters", - "api/types/image", - "api/types/mount", - "api/types/network", - "api/types/registry", - "api/types/strslice", - "api/types/swarm", - "api/types/time", - "api/types/versions", - "api/types/volume", - "client", - "pkg/ioutils", - "pkg/longpath", - "pkg/system", - "pkg/tlsconfig", - ] - pruneopts = "UT" - revision = "90d35abf7b3535c1c319c872900fbd76374e521c" - version = "v17.05.0-ce-rc3" - -[[projects]] - digest = "1:b6b5c3e8da0fb8073cd2886ba249a40f4402b4391ca6eba905a142cceea97a12" - name = "github.com/docker/go-connections" - packages = [ - "nat", - "sockets", - "tlsconfig", - ] - pruneopts = "UT" - revision = "3ede32e2033de7505e6500d6c868c2b9ed9f169d" - version = "v0.3.0" - -[[projects]] - digest = "1:6f82cacd0af5921e99bf3f46748705239b36489464f4529a1589bc895764fb18" - name = "github.com/docker/go-units" - packages = ["."] - pruneopts = "UT" - revision = "47565b4f722fb6ceae66b95f853feed578a4a51c" - version = "v0.3.3" - -[[projects]] - branch = "master" - digest = "1:4841e14252a2cecf11840bd05230412ad469709bbacfc12467e2ce5ad07f339b" - name = "github.com/docker/libtrust" - packages = ["."] - pruneopts = "UT" - revision = "aabc10ec26b754e797f9028f4589c5b7bd90dc20" - -[[projects]] - branch = "master" - digest = "1:67d0b50be0549e610017cb91e0b0b745ec0cad7c613bc8e18ff2d1c1fc8825a7" - name = "github.com/edsrzf/mmap-go" - packages = ["."] - pruneopts = "UT" - revision = "0bce6a6887123b67a60366d2c9fe2dfb74289d2e" - -[[projects]] - digest = "1:3a1c71661e5d956a3c3ff639dba511871757b5f37cb2107abf717d631ece8b1a" - name = "github.com/fsouza/fake-gcs-server" - packages = ["fakestorage"] - pruneopts = "UT" - revision = "9162fe06459e3d9859987fc2802fa729ddb6dc53" - version = "v1.0.0" - -[[projects]] - digest = "1:fe8a03a8222d5b913f256972933d26d24ad7c8286692a42943bc01633cc8fce3" - name = "github.com/go-ini/ini" - packages = ["."] - pruneopts = "UT" - revision = "358ee7663966325963d4e8b2e1fbd570c5195153" - version = "v1.38.1" - -[[projects]] - digest = "1:adea5a94903eb4384abef30f3d878dc9ff6b6b5b0722da25b82e5169216dfb61" - name = "github.com/go-sql-driver/mysql" - packages = ["."] - pruneopts = "UT" - revision = "d523deb1b23d913de5bdada721a6071e71283618" - version = "v1.4.0" - -[[projects]] - branch = "master" - digest = "1:2b43ef0cd185eb036ccd68b629c33aeb10d4ee3264a3912ca25ef1a5ade9c439" - name = "github.com/gocql/gocql" - packages = [ - ".", - "internal/lru", - "internal/murmur", - "internal/streams", - ] - pruneopts = "UT" - revision = "e06f8c1bcd787e6bf0608288b314522f08cc7848" - -[[projects]] - digest = "1:56a4c5b0ea79d81320468aac2c43c0f4295181dce20148fef2227c40f2b34644" - name = "github.com/golang-migrate/migrate" - packages = [ - ".", - "database", - "database/cassandra", - "database/clickhouse", - "database/cockroachdb", - "database/mysql", - "database/postgres", - "database/ql", - "database/redshift", - "database/spanner", - "database/sqlite3", - "database/stub", - "database/testing", - "source", - "source/aws_s3", - "source/file", - "source/github", - "source/go_bindata", - "source/go_bindata/testdata", - "source/godoc_vfs", - "source/google_cloud_storage", - "source/stub", - "source/testing", - "testing", - ] - pruneopts = "UT" - revision = "93d53a5ae84d81945eedadfe8f6865530d61d51c" - version = "v3.5.1" - -[[projects]] - digest = "1:d6d8ef66dbf92848883e228323ca4ee7404eb4fe3681c0e0395f515f128e0d29" - name = "github.com/golang/protobuf" - packages = [ - "proto", - "protoc-gen-go/descriptor", - "ptypes", - "ptypes/any", - "ptypes/duration", - "ptypes/empty", - "ptypes/struct", - "ptypes/timestamp", - ] - pruneopts = "UT" - revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265" - version = "v1.1.0" - -[[projects]] - branch = "master" - digest = "1:4a0c6bb4805508a6287675fac876be2ac1182539ca8a32468d8128882e9d5009" - name = "github.com/golang/snappy" - packages = ["."] - pruneopts = "UT" - revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a" - -[[projects]] - digest = "1:51bee9f1987dcdb9f9a1b4c20745d78f6bf6f5f14ad4e64ca883eb64df4c0045" - name = "github.com/google/go-github" - packages = ["github"] - pruneopts = "UT" - revision = "e48060a28fac52d0f1cb758bc8b87c07bac4a87d" - version = "v15.0.0" - -[[projects]] - branch = "master" - digest = "1:a63cff6b5d8b95638bfe300385d93b2a6d9d687734b863da8e09dc834510a690" - name = "github.com/google/go-querystring" - packages = ["query"] - pruneopts = "UT" - revision = "53e6ce116135b80d037921a7fdd5138cf32d7a8a" - -[[projects]] - digest = "1:e145e9710a10bc114a6d3e2738aadf8de146adaa031854ffdf7bbfe15da85e63" - name = "github.com/googleapis/gax-go" - packages = ["."] - pruneopts = "UT" - revision = "317e0006254c44a0ac427cc52a0e083ff0b9622f" - version = "v2.0.0" - -[[projects]] - digest = "1:c79fb010be38a59d657c48c6ba1d003a8aa651fa56b579d959d74573b7dff8e1" - name = "github.com/gorilla/context" - packages = ["."] - pruneopts = "UT" - revision = "08b5f424b9271eedf6f9f0ce86cb9396ed337a42" - version = "v1.1.1" - -[[projects]] - digest = "1:e73f5b0152105f18bc131fba127d9949305c8693f8a762588a82a48f61756f5f" - name = "github.com/gorilla/mux" - packages = ["."] - pruneopts = "UT" - revision = "e3702bed27f0d39777b0b37b664b6280e8ef8fbf" - version = "v1.6.2" - -[[projects]] - branch = "master" - digest = "1:364b908b9b27b97ab838f2f6f1b1f46281fa29b978a037d72a9b1d4f6d940190" - name = "github.com/hailocab/go-hostpool" - packages = ["."] - pruneopts = "UT" - revision = "e80d13ce29ede4452c43dea11e79b9bc8a15b478" - -[[projects]] - digest = "1:e22af8c7518e1eab6f2eab2b7d7558927f816262586cd6ed9f349c97a6c285c4" - name = "github.com/jmespath/go-jmespath" - packages = ["."] - pruneopts = "UT" - revision = "0b12d6b5" - -[[projects]] - digest = "1:729cb96e4ff32f992aab19a58384fc4304c05f414a9169ef46b8c6ebb5515f77" - name = "github.com/kshvakov/clickhouse" - packages = [ - ".", - "lib/binary", - "lib/column", - "lib/data", - "lib/protocol", - "lib/types", - "lib/writebuffer", - ] - pruneopts = "UT" - revision = "8a2dd1e831a6c7381c11d40bcaf80ae98023c605" - version = "v1.3.3" - -[[projects]] - branch = "master" - digest = "1:37ce7d7d80531b227023331002c0d42b4b4b291a96798c82a049d03a54ba79e4" - name = "github.com/lib/pq" - packages = [ - ".", - "oid", - ] - pruneopts = "UT" - revision = "90697d60dd844d5ef6ff15135d0203f65d2f53b8" - -[[projects]] - digest = "1:3cafc6a5a1b8269605d9df4c6956d43d8011fc57f266ca6b9d04da6c09dee548" - name = "github.com/mattn/go-sqlite3" - packages = ["."] - pruneopts = "UT" - revision = "25ecb14adfc7543176f7d85291ec7dba82c6f7e4" - version = "v1.9.0" - -[[projects]] - digest = "1:ee4d4af67d93cc7644157882329023ce9a7bcfce956a079069a9405521c7cc8d" - name = "github.com/opencontainers/go-digest" - packages = ["."] - pruneopts = "UT" - revision = "279bed98673dd5bef374d3b6e4b09e2af76183bf" - version = "v1.0.0-rc1" - -[[projects]] - digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747" - name = "github.com/pkg/errors" - packages = ["."] - pruneopts = "UT" - revision = "645ef00459ed84a119197bfb8d8205042c6df63d" - version = "v0.8.0" - -[[projects]] - digest = "1:4ed20c76a0526d584d895c0a21c518c1d5a84ff97bd3c5828f3682016445ed39" - name = "go.opencensus.io" - packages = [ - ".", - "exporter/stackdriver/propagation", - "internal", - "internal/tagencoding", - "plugin/ocgrpc", - "plugin/ochttp", - "plugin/ochttp/propagation/b3", - "stats", - "stats/internal", - "stats/view", - "tag", - "trace", - "trace/internal", - "trace/propagation", - ] - pruneopts = "UT" - revision = "e262766cd0d230a1bb7c37281e345e465f19b41b" - version = "v0.14.0" - -[[projects]] - branch = "master" - digest = "1:3f3a05ae0b95893d90b9b3b5afdb79a9b3d96e4e36e099d841ae602e4aca0da8" - name = "golang.org/x/crypto" - packages = ["ssh/terminal"] - pruneopts = "UT" - revision = "c126467f60eb25f8f27e5a981f32a87e3965053f" - -[[projects]] - branch = "master" - digest = "1:4d337ebc84e0ed0205f84f919934e70d830df9c86fc429ca8ce7ac286e29d247" - name = "golang.org/x/net" - packages = [ - "context", - "context/ctxhttp", - "http/httpguts", - "http2", - "http2/hpack", - "idna", - "internal/socks", - "internal/timeseries", - "proxy", - "trace", - ] - pruneopts = "UT" - revision = "3673e40ba22529d22c3fd7c93e97b0ce50fa7bdd" - -[[projects]] - branch = "master" - digest = "1:bea0314c10bd362ab623af4880d853b5bad3b63d0ab9945c47e461b8d04203ed" - name = "golang.org/x/oauth2" - packages = [ - ".", - "google", - "internal", - "jws", - "jwt", - ] - pruneopts = "UT" - revision = "3d292e4d0cdc3a0113e6d207bb137145ef1de42f" - -[[projects]] - branch = "master" - digest = "1:be95d758fc4d9216e5d41ff5b98b46938fba85ca5bc2ddc45a1b03e2bb10fe7c" - name = "golang.org/x/sys" - packages = [ - "unix", - "windows", - ] - pruneopts = "UT" - revision = "e072cadbbdc8dd3d3ffa82b8b4b9304c261d9311" - -[[projects]] - digest = "1:a2ab62866c75542dd18d2b069fec854577a20211d7c0ea6ae746072a1dccdd18" - name = "golang.org/x/text" - packages = [ - "collate", - "collate/build", - "internal/colltab", - "internal/gen", - "internal/tag", - "internal/triegen", - "internal/ucd", - "language", - "secure/bidirule", - "transform", - "unicode/bidi", - "unicode/cldr", - "unicode/norm", - "unicode/rangetable", - ] - pruneopts = "UT" - revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" - version = "v0.3.0" - -[[projects]] - branch = "master" - digest = "1:7db86209b441978ca91b574ec2ec5a12a6c39bc89b659ca667d99eb96521d587" - name = "golang.org/x/tools" - packages = [ - "godoc/vfs", - "godoc/vfs/mapfs", - ] - pruneopts = "UT" - revision = "526516e9c4bf5a3f76fe506dec01ca51d04cbe50" - -[[projects]] - branch = "master" - digest = "1:2892dda36e78635b8ef9e64be5af673150ff9a6c5e3946e9102353e9f01d8b81" - name = "google.golang.org/api" - packages = [ - "gensupport", - "googleapi", - "googleapi/internal/uritemplates", - "googleapi/transport", - "internal", - "iterator", - "option", - "storage/v1", - "transport", - "transport/grpc", - "transport/http", - ] - pruneopts = "UT" - revision = "2c45710c7f3fb0ab63506810a1ba84325ab90ab8" - -[[projects]] - digest = "1:0e781d9592d20f30c4280da30f27b448b4815215f9b43b5258b763a79fb98e98" - name = "google.golang.org/appengine" - packages = [ - ".", - "cloudsql", - "internal", - "internal/app_identity", - "internal/base", - "internal/datastore", - "internal/log", - "internal/modules", - "internal/remote_api", - "internal/socket", - "internal/urlfetch", - "socket", - "urlfetch", - ] - pruneopts = "UT" - revision = "b1f26356af11148e710935ed1ac8a7f5702c7612" - version = "v1.1.0" - -[[projects]] - branch = "master" - digest = "1:7381a470b84bba1d4ded5a158604b04bd002dbb265a63f416a9329bfeee095c7" - name = "google.golang.org/genproto" - packages = [ - "googleapis/api/annotations", - "googleapis/iam/v1", - "googleapis/longrunning", - "googleapis/rpc/code", - "googleapis/rpc/errdetails", - "googleapis/rpc/status", - "googleapis/spanner/admin/database/v1", - "googleapis/spanner/v1", - ] - pruneopts = "UT" - revision = "02b4e95473316948020af0b7a4f0f22c73929b0e" - -[[projects]] - digest = "1:686525281321f81747cfb0bba0bc384511bbdc5cdc92a6aa1afe5f1808f82c36" - name = "google.golang.org/grpc" - packages = [ - ".", - "balancer", - "balancer/base", - "balancer/roundrobin", - "codes", - "connectivity", - "credentials", - "credentials/oauth", - "encoding", - "encoding/proto", - "grpclog", - "internal", - "internal/backoff", - "internal/channelz", - "internal/grpcrand", - "keepalive", - "metadata", - "naming", - "peer", - "resolver", - "resolver/dns", - "resolver/passthrough", - "stats", - "status", - "tap", - "transport", - ] - pruneopts = "UT" - revision = "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" - version = "v1.13.0" - -[[projects]] - digest = "1:2d1fbdc6777e5408cabeb02bf336305e724b925ff4546ded0fa8715a7267922a" - name = "gopkg.in/inf.v0" - packages = ["."] - pruneopts = "UT" - revision = "d2d2541c53f18d2a059457998ce2876cc8e67cbf" - version = "v0.9.1" - -[solve-meta] - analyzer-name = "dep" - analyzer-version = 1 - input-imports = [ - "cloud.google.com/go/spanner", - "cloud.google.com/go/spanner/admin/database/apiv1", - "cloud.google.com/go/storage", - "github.com/Sirupsen/logrus", - "github.com/aws/aws-sdk-go/aws", - "github.com/aws/aws-sdk-go/aws/session", - "github.com/aws/aws-sdk-go/service/s3", - "github.com/aws/aws-sdk-go/service/s3/s3iface", - "github.com/cockroachdb/cockroach-go/crdb", - "github.com/cznic/ql/driver", - "github.com/docker/docker/api/types", - "github.com/docker/docker/api/types/container", - "github.com/docker/docker/api/types/network", - "github.com/docker/docker/client", - "github.com/fsouza/fake-gcs-server/fakestorage", - "github.com/go-sql-driver/mysql", - "github.com/gocql/gocql", - "github.com/golang-migrate/migrate", - "github.com/golang-migrate/migrate/database", - "github.com/golang-migrate/migrate/database/cassandra", - "github.com/golang-migrate/migrate/database/clickhouse", - "github.com/golang-migrate/migrate/database/cockroachdb", - "github.com/golang-migrate/migrate/database/mysql", - "github.com/golang-migrate/migrate/database/postgres", - "github.com/golang-migrate/migrate/database/ql", - "github.com/golang-migrate/migrate/database/redshift", - "github.com/golang-migrate/migrate/database/spanner", - "github.com/golang-migrate/migrate/database/sqlite3", - "github.com/golang-migrate/migrate/database/stub", - "github.com/golang-migrate/migrate/database/testing", - "github.com/golang-migrate/migrate/source", - "github.com/golang-migrate/migrate/source/aws_s3", - "github.com/golang-migrate/migrate/source/file", - "github.com/golang-migrate/migrate/source/github", - "github.com/golang-migrate/migrate/source/go_bindata", - "github.com/golang-migrate/migrate/source/go_bindata/testdata", - "github.com/golang-migrate/migrate/source/godoc_vfs", - "github.com/golang-migrate/migrate/source/google_cloud_storage", - "github.com/golang-migrate/migrate/source/stub", - "github.com/golang-migrate/migrate/source/testing", - "github.com/golang-migrate/migrate/testing", - "github.com/google/go-github/github", - "github.com/kshvakov/clickhouse", - "github.com/lib/pq", - "github.com/mattn/go-sqlite3", - "golang.org/x/net/context", - "golang.org/x/tools/godoc/vfs", - "golang.org/x/tools/godoc/vfs/mapfs", - "google.golang.org/api/iterator", - "google.golang.org/genproto/googleapis/spanner/admin/database/v1", - ] - solver-name = "gps-cdcl" - solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml deleted file mode 100644 index e4343175a..000000000 --- a/Gopkg.toml +++ /dev/null @@ -1,98 +0,0 @@ -# Gopkg.toml example -# -# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html -# for detailed Gopkg.toml documentation. -# -# required = ["github.com/user/thing/cmd/thing"] -# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] -# -# [[constraint]] -# name = "github.com/user/project" -# version = "1.0.0" -# -# [[constraint]] -# name = "github.com/user/project2" -# branch = "dev" -# source = "github.com/myfork/project2" -# -# [[override]] -# name = "github.com/x/y" -# version = "2.4.0" -# -# [prune] -# non-go = false -# go-tests = true -# unused-packages = true - - -[[constraint]] - name = "github.com/aws/aws-sdk-go" - version = "1.15.34" - -[[constraint]] - branch = "master" - name = "github.com/cockroachdb/cockroach-go" - -[[constraint]] - name = "github.com/cznic/ql" - version = "1.2.0" - -[[constraint]] - name = "github.com/docker/docker" - version = "v17.05.0-ce" - -[[constraint]] - name = "github.com/fsouza/fake-gcs-server" - version = "1.2.0" - -[[constraint]] - name = "github.com/go-sql-driver/mysql" - version = "v1.4.0" - -[[constraint]] - branch = "master" - name = "github.com/gocql/gocql" - -[[constraint]] - name = "github.com/google/go-github" - version = "17.0.0" - -[[constraint]] - name = "github.com/kshvakov/clickhouse" - version = "1.3.4" - -[[constraint]] - name = "github.com/lib/pq" - version = "1.0.0" - -[[constraint]] - name = "github.com/mattn/go-sqlite3" - version = "1.9.0" - -[[constraint]] - branch = "master" - name = "golang.org/x/net" - -[[constraint]] - branch = "master" - name = "google.golang.org/api" - -[[constraint]] - branch = "master" - name = "google.golang.org/genproto" - -[[override]] - name = "cloud.google.com/go" - version = "0.27.0" - -[[override]] - branch = "master" - name = "golang.org/x/tools" - -[[override]] - branch = "master" - name = "github.com/docker/distribution" - -[prune] - go-tests = true - unused-packages = true diff --git a/LICENSE b/LICENSE index 62efa3670..d03742c9f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,9 +1,14 @@ The MIT License (MIT) +Original Work Copyright (c) 2016 Matthias Kadenbach - https://github.com/mattes/migrate +Modified Work +Copyright (c) 2018 Dale Hui +https://github.com/golang-migrate/migrate + + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights diff --git a/MIGRATIONS.md b/MIGRATIONS.md index 836388702..3475d8e4e 100644 --- a/MIGRATIONS.md +++ b/MIGRATIONS.md @@ -44,9 +44,14 @@ It is suggested that the version number of corresponding `up` and `down` migrati files be equivalent for clarity, but they are allowed to differ so long as the relative ordering of the migrations is preserved. -The migration files are permitted to be empty, so in the event that a migration -is a no-op or is irreversible, it is recommended to still include both migration -files, and either leaving them empty or adding a comment as appropriate. +The migration files are permitted to be "empty", in the event that a migration +is a no-op or is irreversible. It is recommended to still include both migration +files by making the whole migration file consist of a comment. +If your database does not support comments, then deleting the migration file will also work. +Note, an actual empty file (e.g. a 0 byte file) may cause issues with your database since migrate +will attempt to run an empty query. In this case, deleting the migration file will also work. +For the rational of this behavior see: +[#244 (comment)](https://github.com/golang-migrate/migrate/issues/244#issuecomment-510758270) ## Migration Content Format diff --git a/Makefile b/Makefile index c3122f258..3279fabc7 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,26 @@ -SOURCE ?= file go_bindata github aws_s3 google_cloud_storage godoc_vfs -DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb clickhouse +SOURCE ?= file go_bindata github github_ee bitbucket aws_s3 google_cloud_storage godoc_vfs gitlab +DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb sqlserver firebird neo4j pgx +DATABASE_TEST ?= $(DATABASE) sqlite sqlite3 sqlcipher BUILD_NUMBER ?= 0 VERSION ?= $(shell git describe --tags --long --dirty=-unsupported 2>/dev/null | cut -c 2-)-j$(BUILD_NUMBER) TEST_FLAGS ?= REPO_OWNER ?= $(shell cd .. && basename "$$(pwd)") COVERAGE_DIR ?= .coverage +build: + CGO_ENABLED=0 go build -ldflags='-X main.Version=$(VERSION)' -tags '$(DATABASE) $(SOURCE)' ./cmd/migrate + +build-docker: + CGO_ENABLED=0 go build -a -o build/migrate.linux-386 -ldflags="-s -w -X main.Version=${VERSION}" -tags "$(DATABASE) $(SOURCE)" ./cmd/migrate build-cli: clean -mkdir ./cli/build - cd ./cli && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o build/migrate.linux-amd64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . - cd ./cli && CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -a -o build/migrate.darwin-amd64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . - cd ./cli && CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -a -o build/migrate.windows-amd64.exe -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . + cd ./cmd/migrate && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o ../../cli/build/migrate.linux-amd64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . + cd ./cmd/migrate && CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -a -o ../../cli/build/migrate.linux-armv7 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . + cd ./cmd/migrate && CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -a -o ../../cli/build/migrate.linux-arm64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . + cd ./cmd/migrate && CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -a -o ../../cli/build/migrate.darwin-amd64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . + cd ./cmd/migrate && CGO_ENABLED=0 GOOS=windows GOARCH=386 go build -a -o ../../cli/build/migrate.windows-386.exe -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . + cd ./cmd/migrate && CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -a -o ../../cli/build/migrate.windows-amd64.exe -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . cd ./cli/build && find . -name 'migrate*' | xargs -I{} tar czf {}.tar.gz {} cd ./cli/build && shasum -a 256 * > sha256sum.txt cat ./cli/build/sha256sum.txt @@ -39,27 +48,14 @@ test-short: test: @-rm -r $(COVERAGE_DIR) @mkdir $(COVERAGE_DIR) - make test-with-flags TEST_FLAGS='-v -race -covermode atomic -coverprofile $$(COVERAGE_DIR)/_$$(RAND).txt -bench=. -benchmem -timeout 20m' - @echo 'mode: atomic' > $(COVERAGE_DIR)/combined.txt - @cat $(COVERAGE_DIR)/_*.txt | grep -v 'mode: atomic' >> $(COVERAGE_DIR)/combined.txt + make test-with-flags TEST_FLAGS='-v -race -covermode atomic -coverprofile $$(COVERAGE_DIR)/combined.txt -bench=. -benchmem -timeout 20m' test-with-flags: @echo SOURCE: $(SOURCE) - @echo DATABASE: $(DATABASE) - - @go test $(TEST_FLAGS) . - @go test $(TEST_FLAGS) ./cli/... - @go test $(TEST_FLAGS) ./database - @go test $(TEST_FLAGS) ./testing/... + @echo DATABASE_TEST: $(DATABASE_TEST) - @echo -n '$(SOURCE)' | tr -s ' ' '\n' | xargs -I{} go test $(TEST_FLAGS) ./source/{} - @go test $(TEST_FLAGS) ./source/testing/... - @go test $(TEST_FLAGS) ./source/stub/... - - @echo -n '$(DATABASE)' | tr -s ' ' '\n' | xargs -I{} go test $(TEST_FLAGS) ./database/{} - @go test $(TEST_FLAGS) ./database/testing/... - @go test $(TEST_FLAGS) ./database/stub/... + @go test $(TEST_FLAGS) ./... kill-orphaned-docker-containers: @@ -114,6 +110,12 @@ release: git tag v$(V) @read -p "Press enter to confirm and push to origin ..." && git push origin v$(V) +echo-source: + @echo "$(SOURCE)" + +echo-database: + @echo "$(DATABASE)" + define external_deps @echo '-- $(1)'; go list -f '{{join .Deps "\n"}}' $(1) | grep -v github.com/$(REPO_OWNER)/migrate | xargs go list -f '{{if not .Standard}}{{.ImportPath}}{{end}}' @@ -121,9 +123,9 @@ define external_deps endef -.PHONY: build-cli clean test-short test test-with-flags html-coverage \ +.PHONY: build build-docker build-cli clean test-short test test-with-flags html-coverage \ restore-import-paths rewrite-import-paths list-external-deps release \ - docs kill-docs open-docs kill-orphaned-docker-containers + docs kill-docs open-docs kill-orphaned-docker-containers echo-source echo-database -SHELL = /bin/bash +SHELL = /bin/sh RAND = $(shell echo $$RANDOM) diff --git a/README.md b/README.md index d7284c160..a955c5e7d 100644 --- a/README.md +++ b/README.md @@ -1,56 +1,58 @@ -[![Build Status](https://img.shields.io/travis/golang-migrate/migrate/master.svg)](https://travis-ci.org/golang-migrate/migrate) -[![GoDoc](https://godoc.org/github.com/golang-migrate/migrate?status.svg)](https://godoc.org/github.com/golang-migrate/migrate) +[![GitHub Workflow Status (branch)](https://img.shields.io/github/workflow/status/golang-migrate/migrate/CI/master)](https://github.com/golang-migrate/migrate/actions/workflows/ci.yaml?query=branch%3Amaster) +[![GoDoc](https://pkg.go.dev/badge/github.com/golang-migrate/migrate)](https://pkg.go.dev/github.com/golang-migrate/migrate/v4) [![Coverage Status](https://img.shields.io/coveralls/github/golang-migrate/migrate/master.svg)](https://coveralls.io/github/golang-migrate/migrate?branch=master) [![packagecloud.io](https://img.shields.io/badge/deb-packagecloud.io-844fec.svg)](https://packagecloud.io/golang-migrate/migrate?filter=debs) [![Docker Pulls](https://img.shields.io/docker/pulls/migrate/migrate.svg)](https://hub.docker.com/r/migrate/migrate/) -![Supported Go Versions](https://img.shields.io/badge/Go-1.10%2C%201.11-lightgrey.svg) +![Supported Go Versions](https://img.shields.io/badge/Go-1.16%2C%201.17-lightgrey.svg) [![GitHub Release](https://img.shields.io/github/release/golang-migrate/migrate.svg)](https://github.com/golang-migrate/migrate/releases) - +[![Go Report Card](https://goreportcard.com/badge/github.com/golang-migrate/migrate)](https://goreportcard.com/report/github.com/golang-migrate/migrate) # migrate __Database migrations written in Go. Use as [CLI](#cli-usage) or import as [library](#use-in-your-go-project).__ - * Migrate reads migrations from [sources](#migration-sources) +* Migrate reads migrations from [sources](#migration-sources) and applies them in correct order to a [database](#databases). - * Drivers are "dumb", migrate glues everything together and makes sure the logic is bulletproof. +* Drivers are "dumb", migrate glues everything together and makes sure the logic is bulletproof. (Keeps the drivers lightweight, too.) - * Database drivers don't assume things or try to correct user input. When in doubt, fail. - - -Looking for [v1](https://github.com/golang-migrate/migrate/tree/v1)? +* Database drivers don't assume things or try to correct user input. When in doubt, fail. Forked from [mattes/migrate](https://github.com/mattes/migrate) - ## Databases Database drivers run migrations. [Add a new database?](database/driver.go) - * [PostgreSQL](database/postgres) - * [Redshift](database/redshift) - * [Ql](database/ql) - * [Cassandra](database/cassandra) - * [SQLite](database/sqlite3) ([todo #165](https://github.com/mattes/migrate/issues/165)) - * [MySQL/ MariaDB](database/mysql) - * [Neo4j](database/neo4j) ([todo #167](https://github.com/mattes/migrate/issues/167)) - * [MongoDB](database/mongodb) ([todo #169](https://github.com/mattes/migrate/issues/169)) - * [CrateDB](database/crate) ([todo #170](https://github.com/mattes/migrate/issues/170)) - * [Shell](database/shell) ([todo #171](https://github.com/mattes/migrate/issues/171)) - * [Google Cloud Spanner](database/spanner) - * [CockroachDB](database/cockroachdb) - * [ClickHouse](database/clickhouse) +* [PostgreSQL](database/postgres) +* [PGX](database/pgx) +* [Redshift](database/redshift) +* [Ql](database/ql) +* [Cassandra](database/cassandra) +* [SQLite](database/sqlite) +* [SQLite3](database/sqlite3) ([todo #165](https://github.com/mattes/migrate/issues/165)) +* [SQLCipher](database/sqlcipher) +* [MySQL/ MariaDB](database/mysql) +* [Neo4j](database/neo4j) +* [MongoDB](database/mongodb) +* [CrateDB](database/crate) ([todo #170](https://github.com/mattes/migrate/issues/170)) +* [Shell](database/shell) ([todo #171](https://github.com/mattes/migrate/issues/171)) +* [Google Cloud Spanner](database/spanner) +* [CockroachDB](database/cockroachdb) +* [ClickHouse](database/clickhouse) +* [Firebird](database/firebird) +* [MS SQL Server](database/sqlserver) ### Database URLs -Database connection strings are specified via URLs. The URL format is driver dependent but generally has the form: `dbdriver://username:password@host:port/dbname?option1=true&option2=false` +Database connection strings are specified via URLs. The URL format is driver dependent but generally has the form: `dbdriver://username:password@host:port/dbname?param1=true¶m2=false` Any [reserved URL characters](https://en.wikipedia.org/wiki/Percent-encoding#Percent-encoding_reserved_characters) need to be escaped. Note, the `%` character also [needs to be escaped](https://en.wikipedia.org/wiki/Percent-encoding#Percent-encoding_the_percent_character) Explicitly, the following characters need to be escaped: `!`, `#`, `$`, `%`, `&`, `'`, `(`, `)`, `*`, `+`, `,`, `/`, `:`, `;`, `=`, `?`, `@`, `[`, `]` -It's easiest to always run the URL parts of your DB connection URL (e.g. username, password, etc) through an URL encoder. See the example Python helpers below: +It's easiest to always run the URL parts of your DB connection URL (e.g. username, password, etc) through an URL encoder. See the example Python snippets below: + ```bash $ python3 -c 'import urllib.parse; print(urllib.parse.quote(input("String to encode: "), ""))' String to encode: FAKEpassword!#$%&'()*+,/:;=?@[] @@ -65,43 +67,45 @@ $ Source drivers read migrations from local or remote sources. [Add a new source?](source/driver.go) - * [Filesystem](source/file) - read from fileystem - * [Go-Bindata](source/go_bindata) - read from embedded binary data ([jteeuwen/go-bindata](https://github.com/jteeuwen/go-bindata)) - * [Github](source/github) - read from remote Github repositories - * [AWS S3](source/aws_s3) - read from Amazon Web Services S3 - * [Google Cloud Storage](source/google_cloud_storage) - read from Google Cloud Platform Storage - - +* [Filesystem](source/file) - read from filesystem +* [Go-Bindata](source/go_bindata) - read from embedded binary data ([jteeuwen/go-bindata](https://github.com/jteeuwen/go-bindata)) +* [pkger](source/pkger) - read from embedded binary data ([markbates/pkger](https://github.com/markbates/pkger)) +* [GitHub](source/github) - read from remote GitHub repositories +* [GitHub Enterprise](source/github_ee) - read from remote GitHub Enterprise repositories +* [Bitbucket](source/bitbucket) - read from remote Bitbucket repositories +* [Gitlab](source/gitlab) - read from remote Gitlab repositories +* [AWS S3](source/aws_s3) - read from Amazon Web Services S3 +* [Google Cloud Storage](source/google_cloud_storage) - read from Google Cloud Platform Storage ## CLI usage - * Simple wrapper around this library. - * Handles ctrl+c (SIGINT) gracefully. - * No config search paths, no config files, no magic ENV var injections. +* Simple wrapper around this library. +* Handles ctrl+c (SIGINT) gracefully. +* No config search paths, no config files, no magic ENV var injections. -__[CLI Documentation](cli)__ +__[CLI Documentation](cmd/migrate)__ -### Basic usage: +### Basic usage -``` +```bash $ migrate -source file://path/to/migrations -database postgres://localhost:5432/database up 2 ``` ### Docker usage -``` -$ docker run -v {{ migration dir }}:/migrations --network host migrate/migrate +```bash +$ docker run -v {{ migration dir }}:/migrations --network host migrate/migrate -path=/migrations/ -database postgres://localhost:5432/database up 2 ``` ## Use in your Go project - * API is stable and frozen for this release (v3.x). - * Uses [dep](https://github.com/golang/dep) to manage dependencies - * To help prevent database corruptions, it supports graceful stops via `GracefulStop chan bool`. - * Bring your own logger. - * Uses `io.Reader` streams internally for low memory overhead. - * Thread-safe and no goroutine leaks. +* API is stable and frozen for this release (v3 & v4). +* Uses [Go modules](https://golang.org/cmd/go/#hdr-Modules__module_versions__and_more) to manage dependencies. +* To help prevent database corruptions, it supports graceful stops via `GracefulStop chan bool`. +* Bring your own logger. +* Uses `io.Reader` streams internally for low memory overhead. +* Thread-safe and no goroutine leaks. __[Go Documentation](https://godoc.org/github.com/golang-migrate/migrate)__ @@ -141,18 +145,35 @@ func main() { } ``` +## Getting started + +Go to [getting started](GETTING_STARTED.md) + +## Tutorials + +* [CockroachDB](database/cockroachdb/TUTORIAL.md) +* [PostgreSQL](database/postgres/TUTORIAL.md) + +(more tutorials to come) + ## Migration files Each migration has an up and down migration. [Why?](FAQ.md#why-two-separate-files-up-and-down-for-a-migration) -``` +```bash 1481574547_create_users_table.up.sql 1481574547_create_users_table.down.sql ``` [Best practices: How to write migrations.](MIGRATIONS.md) +## Versions +Version | Supported? | Import | Notes +--------|------------|--------|------ +**master** | :white_check_mark: | `import "github.com/golang-migrate/migrate/v4"` | New features and bug fixes arrive here first | +**v4** | :white_check_mark: | `import "github.com/golang-migrate/migrate/v4"` | Used for stable releases | +**v3** | :x: | `import "github.com/golang-migrate/migrate"` (with package manager) or `import "gopkg.in/golang-migrate/migrate.v3"` (not recommended) | **DO NOT USE** - No longer supported | ## Development and Contributing @@ -161,8 +182,6 @@ read the [development guide](CONTRIBUTING.md). Also have a look at the [FAQ](FAQ.md). - - --- Looking for alternatives? [https://awesome-go.com/#database](https://awesome-go.com/#database). diff --git a/cli/README.md b/cli/README.md index 15bb62061..cffeb2995 100644 --- a/cli/README.md +++ b/cli/README.md @@ -1,117 +1,3 @@ -# migrate CLI +# Deprecated -## Installation - -#### Download pre-build binary (Windows, MacOS, or Linux) - -[Release Downloads](https://github.com/golang-migrate/migrate/releases) - -``` -$ curl -L https://github.com/golang-migrate/migrate/releases/download/$version/migrate.$platform-amd64.tar.gz | tar xvz -``` - -#### MacOS - -We have not released support for homebrew yet, but there is a live issue here: [todo #156](https://github.com/mattes/migrate/issues/156) - -Any help to make this happen would be appreciated! - -#### Linux (*.deb package) - -``` -$ curl -L https://packagecloud.io/golang-migrate/migrate/gpgkey | apt-key add - -$ echo "deb https://packagecloud.io/golang-migrate/migrate/ubuntu/ xenial main" > /etc/apt/sources.list.d/migrate.list -$ apt-get update -$ apt-get install -y migrate -``` - -#### With Go toolchain - -``` -$ go get -u -d github.com/golang-migrate/migrate/cli -$ cd $GOPATH/src/github.com/golang-migrate/migrate/cli -$ dep ensure -$ go build -tags 'postgres' -o /usr/local/bin/migrate github.com/golang-migrate/migrate/cli -``` - -##### Notes: -1. This example builds the cli which will only work with postgres. In order -to build the cli for use with other databases, replace the `postgres` build tag -with the appropriate database tag(s) for the databases desired. The tags -correspond to the names of the sub-packages underneath the -[`database`](../database) package. -1. Similarly to the database build tags, if you need to support other sources, use the appropriate build tag(s). -1. Support for build constraints will be removed in the future: https://github.com/golang-migrate/migrate/issues/60 - - -## Usage - -``` -$ migrate --help -Usage: migrate OPTIONS COMMAND [arg...] - migrate [ --version | --help ] - -Options: - --source Location of the migrations (driver://url) - --path Shorthand for -source=file://path - --database Run migrations against this database (driver://url) - --prefetch N Number of migrations to load in advance before executing (default 10) - --lock-timeout N Allow N seconds to acquire database lock (default 15) - --verbose Print verbose logging - --version Print version - --help Print usage - -Commands: - create [-ext E] [-dir D] [-seq] [-digits N] [-format] NAME - Create a set of timestamped up/down migrations titled NAME, in directory D with extension E. - Use -seq option to generate sequential up/down migrations with N digits. - Use -format option to specify a Go time format string. - goto V Migrate to version V - up [N] Apply all or N up migrations - down [N] Apply all or N down migrations - drop Drop everyting inside database - force V Set version V but don't run migration (ignores dirty state) - version Print current migration version -``` - - -So let's say you want to run the first two migrations - -``` -$ migrate --source file://path/to/migrations --database postgres://localhost:5432/database up 2 -``` - -If your migrations are hosted on github - -``` -$ migrate --source github://mattes:personal-access-token@mattes/migrate_test \ - --database postgres://localhost:5432/database down 2 -``` - -The CLI will gracefully stop at a safe point when SIGINT (ctrl+c) is received. -Send SIGKILL for immediate halt. - - - -## Reading CLI arguments from somewhere else - -##### ENV variables - -``` -$ migrate --database "$MY_MIGRATE_DATABASE" -``` - -##### JSON files - -Check out https://stedolan.github.io/jq/ - -``` -$ migrate --database "$(cat config.json | jq '.database')" -``` - -##### YAML files - -```` -$ migrate --database "$(cat config/database.yml | ruby -ryaml -e "print YAML.load(STDIN.read)['database']")" -$ migrate --database "$(cat config/database.yml | python -c 'import yaml,sys;print yaml.safe_load(sys.stdin)["database"]')" -``` +Use [cmd/migrate](../cmd/migrate) instead diff --git a/cli/commands.go b/cli/commands.go deleted file mode 100644 index 11dfbba9e..000000000 --- a/cli/commands.go +++ /dev/null @@ -1,166 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "github.com/golang-migrate/migrate/v4" - _ "github.com/golang-migrate/migrate/v4/database/stub" // TODO remove again - _ "github.com/golang-migrate/migrate/v4/source/file" - "os" - "path/filepath" - "strconv" - "strings" - "time" -) - -func nextSeq(matches []string, dir string, seqDigits int) (string, error) { - if seqDigits <= 0 { - return "", errors.New("Digits must be positive") - } - - nextSeq := 1 - if len(matches) > 0 { - filename := matches[len(matches)-1] - matchSeqStr := strings.TrimPrefix(filename, dir) - idx := strings.Index(matchSeqStr, "_") - if idx < 1 { // Using 1 instead of 0 since there should be at least 1 digit - return "", errors.New("Malformed migration filename: " + filename) - } - matchSeqStr = matchSeqStr[0:idx] - var err error - nextSeq, err = strconv.Atoi(matchSeqStr) - if err != nil { - return "", err - } - nextSeq++ - } - if nextSeq <= 0 { - return "", errors.New("Next sequence number must be positive") - } - - nextSeqStr := strconv.Itoa(nextSeq) - if len(nextSeqStr) > seqDigits { - return "", fmt.Errorf("Next sequence number %s too large. At most %d digits are allowed", nextSeqStr, seqDigits) - } - padding := seqDigits - len(nextSeqStr) - if padding > 0 { - nextSeqStr = strings.Repeat("0", padding) + nextSeqStr - } - return nextSeqStr, nil -} - -func createCmd(dir string, startTime time.Time, format string, name string, ext string, seq bool, seqDigits int) { - var base string - if seq && format != defaultTimeFormat { - log.fatalErr(errors.New("The seq and format options are mutually exclusive")) - } - if seq { - if seqDigits <= 0 { - log.fatalErr(errors.New("Digits must be positive")) - } - matches, err := filepath.Glob(dir + "*" + ext) - if err != nil { - log.fatalErr(err) - } - nextSeqStr, err := nextSeq(matches, dir, seqDigits) - if err != nil { - log.fatalErr(err) - } - base = fmt.Sprintf("%v%v_%v.", dir, nextSeqStr, name) - } else { - switch format { - case "": - log.fatal("Time format may not be empty") - case "unix": - base = fmt.Sprintf("%v%v_%v.", dir, startTime.Unix(), name) - case "unixNano": - base = fmt.Sprintf("%v%v_%v.", dir, startTime.UnixNano(), name) - default: - base = fmt.Sprintf("%v%v_%v.", dir, startTime.Format(format), name) - } - } - - os.MkdirAll(dir, os.ModePerm) - createFile(base + "up" + ext) - createFile(base + "down" + ext) -} - -func createFile(fname string) { - if _, err := os.Create(fname); err != nil { - log.fatalErr(err) - } -} - -func gotoCmd(m *migrate.Migrate, v uint) { - if err := m.Migrate(v); err != nil { - if err != migrate.ErrNoChange { - log.fatalErr(err) - } else { - log.Println(err) - } - } -} - -func upCmd(m *migrate.Migrate, limit int) { - if limit >= 0 { - if err := m.Steps(limit); err != nil { - if err != migrate.ErrNoChange { - log.fatalErr(err) - } else { - log.Println(err) - } - } - } else { - if err := m.Up(); err != nil { - if err != migrate.ErrNoChange { - log.fatalErr(err) - } else { - log.Println(err) - } - } - } -} - -func downCmd(m *migrate.Migrate, limit int) { - if limit >= 0 { - if err := m.Steps(-limit); err != nil { - if err != migrate.ErrNoChange { - log.fatalErr(err) - } else { - log.Println(err) - } - } - } else { - if err := m.Down(); err != nil { - if err != migrate.ErrNoChange { - log.fatalErr(err) - } else { - log.Println(err) - } - } - } -} - -func dropCmd(m *migrate.Migrate) { - if err := m.Drop(); err != nil { - log.fatalErr(err) - } -} - -func forceCmd(m *migrate.Migrate, v int) { - if err := m.Force(v); err != nil { - log.fatalErr(err) - } -} - -func versionCmd(m *migrate.Migrate) { - v, dirty, err := m.Version() - if err != nil { - log.fatalErr(err) - } - if dirty { - log.Printf("%v (dirty)\n", v) - } else { - log.Println(v) - } -} diff --git a/cli/commands_test.go b/cli/commands_test.go deleted file mode 100644 index 6ab0ed44b..000000000 --- a/cli/commands_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package main - -import ( - "testing" -) - -func TestNextSeq(t *testing.T) { - cases := []struct { - name string - matches []string - dir string - seqDigits int - expected string - expectedErrStr string - }{ - {"Bad digits", []string{}, "migrationDir", 0, "", "Digits must be positive"}, - {"Single digit initialize", []string{}, "migrationDir", 1, "1", ""}, - {"Single digit malformed", []string{"bad"}, "migrationDir", 1, "", "Malformed migration filename: bad"}, - {"Single digit no int", []string{"bad_bad"}, "migrationDir", 1, "", "strconv.Atoi: parsing \"bad\": invalid syntax"}, - {"Single digit negative seq", []string{"-5_test"}, "migrationDir", 1, "", "Next sequence number must be positive"}, - {"Single digit increment", []string{"3_test", "4_test"}, "migrationDir", 1, "5", ""}, - {"Single digit overflow", []string{"9_test"}, "migrationDir", 1, "", "Next sequence number 10 too large. At most 1 digits are allowed"}, - {"Zero-pad initialize", []string{}, "migrationDir", 6, "000001", ""}, - {"Zero-pad malformed", []string{"bad"}, "migrationDir", 6, "", "Malformed migration filename: bad"}, - {"Zero-pad no int", []string{"bad_bad"}, "migrationDir", 6, "", "strconv.Atoi: parsing \"bad\": invalid syntax"}, - {"Zero-pad negative seq", []string{"-000005_test"}, "migrationDir", 6, "", "Next sequence number must be positive"}, - {"Zero-pad increment", []string{"000003_test", "000004_test"}, "migrationDir", 6, "000005", ""}, - {"Zero-pad overflow", []string{"999999_test"}, "migrationDir", 6, "", "Next sequence number 1000000 too large. At most 6 digits are allowed"}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - nextSeq, err := nextSeq(c.matches, c.dir, c.seqDigits) - if nextSeq != c.expected { - t.Error("Incorrect nextSeq: " + nextSeq + " != " + c.expected) - } - if err != nil { - if err.Error() != c.expectedErrStr { - t.Error("Incorrect error: " + err.Error() + " != " + c.expectedErrStr) - } - } else if c.expectedErrStr != "" { - t.Error("Expected error: " + c.expectedErrStr + " but got nil instead") - } - }) - } -} diff --git a/cli/examples/Dockerfile b/cli/examples/Dockerfile deleted file mode 100644 index f6cd90c58..000000000 --- a/cli/examples/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM ubuntu:xenial - -RUN apt-get update && \ - apt-get install -y curl apt-transport-https - -RUN curl -L https://packagecloud.io/golang-migrate/migrate/gpgkey | apt-key add - && \ - echo "deb https://packagecloud.io/golang-migrate/migrate/ubuntu/ xenial main" > /etc/apt/sources.list.d/migrate.list && \ - apt-get update && \ - apt-get install -y migrate - -RUN migrate -version - diff --git a/cli/main.go b/cli/main.go index 86065c433..254e8168f 100644 --- a/cli/main.go +++ b/cli/main.go @@ -1,287 +1,10 @@ package main import ( - "fmt" - "net/url" - "os" - "os/signal" - "strconv" - "strings" - "syscall" - "time" - - "github.com/spf13/pflag" - "github.com/spf13/viper" - - "github.com/golang-migrate/migrate/v4" - "github.com/golang-migrate/migrate/v4/database" - "github.com/golang-migrate/migrate/v4/source" + "github.com/golang-migrate/migrate/v4/internal/cli" ) -const defaultTimeFormat = "20060102150405" - -// set main log -var log = &Log{} - -func init() { - pflag.Usage = func() { - fmt.Fprint(os.Stderr, - `Usage: migrate OPTIONS COMMAND [arg...] - migrate [ --version | --help ] - -Options: - --config.source directory of the configuration file (default "/cli/config") - --config.file configuration file name (without extension) - --database.dsn database connection string - --database.driver database driver (default postgres) - --database.address address of the database (default "0.0.0.0:5432") - --database.name name of the database - --database.user database username (default "postgres") - --database.password database password (default "postgres") - --database.ssl database ssl mode (default "disable") - --path Shorthand for -source=file://path - --source Location of the migrations (driver://url) - --lock-timeout Allow N seconds to acquire database lock (default 15) - --prefetch Number of migrations to load in advance before executing (default 10) - --verbose Print verbose logging (default true) - --version Print version - --help Print usage - -Commands: - create [-ext E] [-dir D] [-seq] [-digits N] [-format] NAME - Create a set of timestamped up/down migrations titled NAME, in directory D with extension E. - Use -seq option to generate sequential up/down migrations with N digits. - Use -format option to specify a Go time format string. - goto V Migrate to version V - up [N] Apply all or N up migrations - down [N] Apply all or N down migrations - drop Drop everyting inside database - force V Set version V but don't run migration (ignores dirty state) - version Print current migration version - -Source drivers: `+strings.Join(source.List(), ", ")+` -Database drivers: `+strings.Join(database.List(), ", ")+"\n") - } - - pflag.Parse() - viper.BindPFlags(pflag.CommandLine) - viper.AutomaticEnv() - viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) - viper.AddConfigPath(viper.GetString("config.source")) - if viper.GetString("config.file") != "" { - viper.SetConfigName(viper.GetString("config.file")) - if err := viper.ReadInConfig(); err != nil { - log.fatalf("cannot load configuration: %v", err) - } - } -} - +// Deprecated, please use cmd/migrate func main() { - help := viper.GetBool("help") - version := viper.GetBool("version") - verbose := viper.GetBool("verbose") - prefetch := viper.GetInt("prefetch") - lockTimeout := viper.GetInt("lock-timeout") - path := viper.GetString("path") - sourcePtr := viper.GetString("source") - - dbSource := viper.GetString("database.dsn") - if dbSource == "" { - dbSource = dbMakeConnectionString( - viper.GetString("database.driver"), viper.GetString("database.user"), - viper.GetString("database.password"), viper.GetString("database.address"), - viper.GetString("database.name"), viper.GetString("database.ssl"), - ) - } - - // initialize logger - log.verbose = verbose - - // show cli version - if version { - fmt.Fprintln(os.Stderr, Version) - os.Exit(0) - } - - // show help - if help { - pflag.Usage() - os.Exit(0) - } - - // translate -path into -source if given - if sourcePtr == "" && path != "" { - sourcePtr = fmt.Sprintf("file://%v", path) - } - - // initialize migrate - // don't catch migraterErr here and let each command decide - // how it wants to handle the error - migrater, migraterErr := migrate.New(sourcePtr, dbSource) - defer func() { - if migraterErr == nil { - migrater.Close() - } - }() - if migraterErr == nil { - migrater.Log = log - migrater.PrefetchMigrations = uint(prefetch) - migrater.LockTimeout = time.Duration(int64(lockTimeout)) * time.Second - - // handle Ctrl+c - signals := make(chan os.Signal, 1) - signal.Notify(signals, syscall.SIGINT) - go func() { - for range signals { - log.Println("Stopping after this running migration ...") - migrater.GracefulStop <- true - return - } - }() - } - - startTime := time.Now() - - switch pflag.Arg(0) { - case "create": - args := pflag.Args()[1:] - seq := false - seqDigits := 6 - - createFlagSet := pflag.NewFlagSet("create", pflag.ExitOnError) - extPtr := createFlagSet.String("ext", "", "File extension") - dirPtr := createFlagSet.String("dir", "", "Directory to place file in (default: current working directory)") - formatPtr := createFlagSet.String("format", defaultTimeFormat, `The Go time format string to use. If the string "unix" or "unixNano" is specified, then the seconds or nanoseconds since January 1, 1970 UTC respectively will be used. Caution, due to the behavior of time.Time.Format(), invalid format strings will not error`) - createFlagSet.BoolVar(&seq, "seq", seq, "Use sequential numbers instead of timestamps (default: false)") - createFlagSet.IntVar(&seqDigits, "digits", seqDigits, "The number of digits to use in sequences (default: 6)") - createFlagSet.Parse(args) - - if createFlagSet.NArg() == 0 { - log.fatal("error: please specify name") - } - name := createFlagSet.Arg(0) - - if *extPtr == "" { - log.fatal("error: -ext flag must be specified") - } - *extPtr = "." + strings.TrimPrefix(*extPtr, ".") - - if *dirPtr != "" { - *dirPtr = strings.Trim(*dirPtr, "/") + "/" - } - - createCmd(*dirPtr, startTime, *formatPtr, name, *extPtr, seq, seqDigits) - - case "goto": - if migraterErr != nil { - log.fatalErr(migraterErr) - } - - if pflag.Arg(1) == "" { - log.fatal("error: please specify version argument V") - } - - v, err := strconv.ParseUint(pflag.Arg(1), 10, 64) - if err != nil { - log.fatal("error: can't read version argument V") - } - - gotoCmd(migrater, uint(v)) - - if log.verbose { - log.Println("Finished after", time.Now().Sub(startTime)) - } - - case "up": - if migraterErr != nil { - log.fatalErr(migraterErr) - } - - limit := -1 - if pflag.Arg(1) != "" { - n, err := strconv.ParseUint(pflag.Arg(1), 10, 64) - if err != nil { - log.fatal("error: can't read limit argument N") - } - limit = int(n) - } - - upCmd(migrater, limit) - - if log.verbose { - log.Println("Finished after", time.Now().Sub(startTime)) - } - - case "down": - if migraterErr != nil { - log.fatalErr(migraterErr) - } - - limit := -1 - if pflag.Arg(1) != "" { - n, err := strconv.ParseUint(pflag.Arg(1), 10, 64) - if err != nil { - log.fatal("error: can't read limit argument N") - } - limit = int(n) - } - - downCmd(migrater, limit) - - if log.verbose { - log.Println("Finished after", time.Now().Sub(startTime)) - } - - case "drop": - if migraterErr != nil { - log.fatalErr(migraterErr) - } - - dropCmd(migrater) - - if log.verbose { - log.Println("Finished after", time.Now().Sub(startTime)) - } - - case "force": - if migraterErr != nil { - log.fatalErr(migraterErr) - } - - if pflag.Arg(1) == "" { - log.fatal("error: please specify version argument V") - } - - v, err := strconv.ParseInt(pflag.Arg(1), 10, 64) - if err != nil { - log.fatal("error: can't read version argument V") - } - - if v < -1 { - log.fatal("error: argument V must be >= -1") - } - - forceCmd(migrater, int(v)) - - if log.verbose { - log.Println("Finished after", time.Now().Sub(startTime)) - } - - case "version": - if migraterErr != nil { - log.fatalErr(migraterErr) - } - - versionCmd(migrater) - - default: - pflag.Usage() - os.Exit(0) - } -} - -func dbMakeConnectionString(driver, user, password, address, name, ssl string) string { - return fmt.Sprintf("%s://%s:%s@%s/%s?sslmode=%s", - driver, url.QueryEscape(user), url.QueryEscape(password), address, name, ssl, - ) + cli.Main(Version) } diff --git a/cmd/migrate/README.md b/cmd/migrate/README.md new file mode 100644 index 000000000..ed9c99b08 --- /dev/null +++ b/cmd/migrate/README.md @@ -0,0 +1,138 @@ +# migrate CLI + +## Installation + +### Download pre-built binary (Windows, MacOS, or Linux) + +[Release Downloads](https://github.com/golang-migrate/migrate/releases) + +```bash +$ curl -L https://github.com/golang-migrate/migrate/releases/download/$version/migrate.$platform-amd64.tar.gz | tar xvz +``` + +### MacOS + +```bash +$ brew install golang-migrate +``` + +### Windows + +Using [scoop](https://scoop.sh/) + +```bash +$ scoop install migrate +``` + +### Linux (*.deb package) + +```bash +$ curl -L https://packagecloud.io/golang-migrate/migrate/gpgkey | apt-key add - +$ echo "deb https://packagecloud.io/golang-migrate/migrate/ubuntu/ $(lsb_release -sc) main" > /etc/apt/sources.list.d/migrate.list +$ apt-get update +$ apt-get install -y migrate +``` + +### With Go toolchain + +#### Versioned + +```bash +$ go get -u -d github.com/golang-migrate/migrate/cmd/migrate +$ cd $GOPATH/src/github.com/golang-migrate/migrate/cmd/migrate +$ git checkout $TAG # e.g. v4.1.0 +$ # Go 1.15 and below +$ go build -tags 'postgres' -ldflags="-X main.Version=$(git describe --tags)" -o $GOPATH/bin/migrate $GOPATH/src/github.com/golang-migrate/migrate/cmd/migrate +$ # Go 1.16+ +$ go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@$TAG +``` + +#### Unversioned + +```bash +$ # Go 1.15 and below +$ go get -tags 'postgres' -u github.com/golang-migrate/migrate/cmd/migrate +$ # Go 1.16+ +$ go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest +``` + +#### Notes + +1. Requires a version of Go that [supports modules](https://golang.org/cmd/go/#hdr-Preliminary_module_support). e.g. Go 1.11+ +1. These examples build the cli which will only work with postgres. In order +to build the cli for use with other databases, replace the `postgres` build tag +with the appropriate database tag(s) for the databases desired. The tags +correspond to the names of the sub-packages underneath the +[`database`](../database) package. +1. Similarly to the database build tags, if you need to support other sources, use the appropriate build tag(s). +1. Support for build constraints will be removed in the future: https://github.com/golang-migrate/migrate/issues/60 +1. For versions of Go 1.15 and lower, [make sure](https://github.com/golang-migrate/migrate/pull/257#issuecomment-705249902) you're not installing the `migrate` CLI from a module. e.g. there should not be any `go.mod` files in your current directory or any directory from your current directory to the root + +## Usage + +```bash +$ migrate -help +Usage: migrate OPTIONS COMMAND [arg...] + migrate [ -version | -help ] + +Options: + -source Location of the migrations (driver://url) + -path Shorthand for -source=file://path + -database Run migrations against this database (driver://url) + -prefetch N Number of migrations to load in advance before executing (default 10) + -lock-timeout N Allow N seconds to acquire database lock (default 15) + -verbose Print verbose logging + -version Print version + -help Print usage + +Commands: + create [-ext E] [-dir D] [-seq] [-digits N] [-format] NAME + Create a set of timestamped up/down migrations titled NAME, in directory D with extension E. + Use -seq option to generate sequential up/down migrations with N digits. + Use -format option to specify a Go time format string. + goto V Migrate to version V + up [N] Apply all or N up migrations + down [N] Apply all or N down migrations + drop Drop everything inside database + force V Set version V but don't run migration (ignores dirty state) + version Print current migration version +``` + +So let's say you want to run the first two migrations + +```bash +$ migrate -source file://path/to/migrations -database postgres://localhost:5432/database up 2 +``` + +If your migrations are hosted on github + +```bash +$ migrate -source github://mattes:personal-access-token@mattes/migrate_test \ + -database postgres://localhost:5432/database down 2 +``` + +The CLI will gracefully stop at a safe point when SIGINT (ctrl+c) is received. +Send SIGKILL for immediate halt. + +## Reading CLI arguments from somewhere else + +### ENV variables + +```bash +$ migrate -database "$MY_MIGRATE_DATABASE" +``` + +### JSON files + +Check out https://stedolan.github.io/jq/ + +```bash +$ migrate -database "$(cat config.json | jq '.database')" +``` + +### YAML files + +```bash +$ migrate -database "$(cat config/database.yml | ruby -ryaml -e "print YAML.load(STDIN.read)['database']")" +$ migrate -database "$(cat config/database.yml | python -c 'import yaml,sys;print yaml.safe_load(sys.stdin)["database"]')" +``` diff --git a/cli/config.go b/cmd/migrate/config.go similarity index 96% rename from cli/config.go rename to cmd/migrate/config.go index 96c6a6357..e5a946922 100644 --- a/cli/config.go +++ b/cmd/migrate/config.go @@ -17,7 +17,7 @@ const ( var ( // define flag overrides flagHelp = pflag.Bool("help", false, "Print usage") - flagVersion = pflag.Bool("version", false, "Print version") + flagVersion = pflag.String("version", Version, "Print version") flagLoggingVerbose = pflag.Bool("verbose", true, "Print verbose logging") flagPrefetch = pflag.Uint("prefetch", 10, "Number of migrations to load in advance before executing") flaglockTimeout = pflag.Uint("lock-timeout", 15, "Allow N seconds to acquire database lock") diff --git a/cli/config/defaults.yaml b/cmd/migrate/config/defaults.yaml similarity index 100% rename from cli/config/defaults.yaml rename to cmd/migrate/config/defaults.yaml diff --git a/cmd/migrate/examples/Dockerfile b/cmd/migrate/examples/Dockerfile new file mode 100644 index 000000000..c78b32023 --- /dev/null +++ b/cmd/migrate/examples/Dockerfile @@ -0,0 +1,15 @@ +FROM ubuntu:bionic + +RUN apt-get update && \ + apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg-agent + +RUN curl -sSL https://packagecloud.io/golang-migrate/migrate/gpgkey | apt-key add - +RUN echo "deb https://packagecloud.io/golang-migrate/migrate/ubuntu/ bionic main" > /etc/apt/sources.list.d/migrate.list +RUN apt-get update && \ + apt-get install -y migrate + +RUN migrate -version diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go new file mode 100644 index 000000000..a91a3650e --- /dev/null +++ b/cmd/migrate/main.go @@ -0,0 +1,32 @@ +package main + +import ( + "log" + "strings" + + "github.com/golang-migrate/migrate/v4/internal/cli" + "github.com/sirupsen/logrus" + "github.com/spf13/pflag" + "github.com/spf13/viper" +) + +func init() { + pflag.Parse() + viper.BindPFlags(pflag.CommandLine) + viper.AutomaticEnv() + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + viper.AddConfigPath(viper.GetString("config.source")) + if viper.GetString("config.file") != "" { + viper.SetConfigName(viper.GetString("config.file")) + if err := viper.ReadInConfig(); err != nil { + log.Fatalf("cannot load configuration: %v", err) + } + } + // logrus formatter + customFormatter := new(logrus.JSONFormatter) + logrus.SetFormatter(customFormatter) +} + +func main() { + cli.Main(Version) +} diff --git a/cmd/migrate/version.go b/cmd/migrate/version.go new file mode 100644 index 000000000..6c3ec49fe --- /dev/null +++ b/cmd/migrate/version.go @@ -0,0 +1,4 @@ +package main + +// Version is set in Makefile with build flags +var Version = "dev" diff --git a/database/cassandra/README.md b/database/cassandra/README.md index f0e1182fc..c3d4387ad 100644 --- a/database/cassandra/README.md +++ b/database/cassandra/README.md @@ -3,7 +3,7 @@ * Drop command will not work on Cassandra 2.X because it rely on system_schema table which comes with 3.X * Other commands should work properly but are **not tested** -* The Cassandra driver (gocql) does not natively support executing multipe statements in a single query. To allow for multiple statements in a single migration, you can use the `x-multi-statement` param. There are two important caveats: +* The Cassandra driver (gocql) does not natively support executing multiple statements in a single query. To allow for multiple statements in a single migration, you can use the `x-multi-statement` param. There are two important caveats: * This mode splits the migration text into separately-executed statements by a semi-colon `;`. Thus `x-multi-statement` cannot be used when a statement in the migration contains a string with a semi-colon. * The queries are not executed in any sort of transaction/batch, meaning you are responsible for fixing partial migrations. @@ -22,7 +22,11 @@ system_schema table which comes with 3.X | `timeout` | 1 minute | Migration timeout | `username` | nil | Username to use when authenticating. | | `password` | nil | Password to use when authenticating. | - +| `sslcert` | | Cert file location. The file must contain PEM encoded data. | +| `sslkey` | | Key file location. The file must contain PEM encoded data. | +| `sslrootcert` | | The location of the root certificate file. The file must contain PEM encoded data. | +| `sslmode` | | Whether or not to use SSL (disable\|require\|verify-ca\|verify-full) | +| `disable-host-lookup`| false | Disable initial host lookup. | `timeout` is parsed using [time.ParseDuration(s string)](https://golang.org/pkg/time/#ParseDuration) diff --git a/database/cassandra/cassandra.go b/database/cassandra/cassandra.go index 9dbb18195..a639426d1 100644 --- a/database/cassandra/cassandra.go +++ b/database/cassandra/cassandra.go @@ -3,6 +3,7 @@ package cassandra import ( "errors" "fmt" + "go.uber.org/atomic" "io" "io/ioutil" nurl "net/url" @@ -12,6 +13,8 @@ import ( "github.com/gocql/gocql" "github.com/golang-migrate/migrate/v4/database" + "github.com/golang-migrate/migrate/v4/database/multistmt" + "github.com/hashicorp/go-multierror" ) func init() { @@ -19,6 +22,12 @@ func init() { database.Register("cassandra", db) } +var ( + multiStmtDelimiter = []byte(";") + + DefaultMultiStatementMaxSize = 10 * 1 << 20 // 10 MB +) + var DefaultMigrationsTable = "schema_migrations" var ( @@ -32,11 +41,12 @@ type Config struct { MigrationsTable string KeyspaceName string MultiStatementEnabled bool + MultiStatementMaxSize int } type Cassandra struct { session *gocql.Session - isLocked bool + isLocked atomic.Bool // Open and WithInstance need to guarantee that config is never nil config *Config @@ -57,6 +67,10 @@ func WithInstance(session *gocql.Session, config *Config) (database.Driver, erro config.MigrationsTable = DefaultMigrationsTable } + if config.MultiStatementMaxSize <= 0 { + config.MultiStatementMaxSize = DefaultMultiStatementMaxSize + } + c := &Cassandra{ session: session, config: config, @@ -120,15 +134,54 @@ func (c *Cassandra) Open(url string) (database.Driver, error) { cluster.Timeout = timeout } + if len(u.Query().Get("sslmode")) > 0 { + if u.Query().Get("sslmode") != "disable" { + sslOpts := &gocql.SslOptions{} + + if len(u.Query().Get("sslrootcert")) > 0 { + sslOpts.CaPath = u.Query().Get("sslrootcert") + } + if len(u.Query().Get("sslcert")) > 0 { + sslOpts.CertPath = u.Query().Get("sslcert") + } + if len(u.Query().Get("sslkey")) > 0 { + sslOpts.KeyPath = u.Query().Get("sslkey") + } + + if u.Query().Get("sslmode") == "verify-full" { + sslOpts.EnableHostVerification = true + } + + cluster.SslOpts = sslOpts + } + } + + if len(u.Query().Get("disable-host-lookup")) > 0 { + if flag, err := strconv.ParseBool(u.Query().Get("disable-host-lookup")); err != nil && flag { + cluster.DisableInitialHostLookup = true + } else if err != nil { + return nil, err + } + } + session, err := cluster.CreateSession() if err != nil { return nil, err } + multiStatementMaxSize := DefaultMultiStatementMaxSize + if s := u.Query().Get("x-multi-statement-max-size"); len(s) > 0 { + multiStatementMaxSize, err = strconv.Atoi(s) + if err != nil { + return nil, err + } + } + return WithInstance(session, &Config{ KeyspaceName: strings.TrimPrefix(u.Path, "/"), MigrationsTable: u.Query().Get("x-migrations-table"), MultiStatementEnabled: u.Query().Get("x-multi-statement") == "true", + MultiStatementMaxSize: multiStatementMaxSize, }) } @@ -138,44 +191,44 @@ func (c *Cassandra) Close() error { } func (c *Cassandra) Lock() error { - if c.isLocked { + if !c.isLocked.CAS(false, true) { return database.ErrLocked } - c.isLocked = true return nil } func (c *Cassandra) Unlock() error { - c.isLocked = false + if !c.isLocked.CAS(true, false) { + return database.ErrNotLocked + } return nil } func (c *Cassandra) Run(migration io.Reader) error { - migr, err := ioutil.ReadAll(migration) - if err != nil { - return err - } - // run migration - query := string(migr[:]) - if c.config.MultiStatementEnabled { - // split query by semi-colon - queries := strings.Split(query, ";") - - for _, q := range queries { - tq := strings.TrimSpace(q) + var err error + if e := multistmt.Parse(migration, multiStmtDelimiter, c.config.MultiStatementMaxSize, func(m []byte) bool { + tq := strings.TrimSpace(string(m)) if tq == "" { - continue + return true } - if err := c.session.Query(tq).Exec(); err != nil { - // TODO: cast to Cassandra error and get line number - return database.Error{OrigErr: err, Err: "migration failed", Query: migr} + if e := c.session.Query(tq).Exec(); e != nil { + err = database.Error{OrigErr: e, Err: "migration failed", Query: m} + return false } + return true + }); e != nil { + return e } - return nil + return err } - if err := c.session.Query(query).Exec(); err != nil { + migr, err := ioutil.ReadAll(migration) + if err != nil { + return err + } + // run migration + if err := c.session.Query(string(migr)).Exec(); err != nil { // TODO: cast to Cassandra error and get line number return database.Error{OrigErr: err, Err: "migration failed", Query: migr} } @@ -183,12 +236,26 @@ func (c *Cassandra) Run(migration io.Reader) error { } func (c *Cassandra) SetVersion(version int, dirty bool) error { - query := `TRUNCATE "` + c.config.MigrationsTable + `"` - if err := c.session.Query(query).Exec(); err != nil { - return &database.Error{OrigErr: err, Query: []byte(query)} + // DELETE instead of TRUNCATE because AWS Keyspaces does not support it + // see: https://docs.aws.amazon.com/keyspaces/latest/devguide/cassandra-apis.html + squery := `SELECT version FROM "` + c.config.MigrationsTable + `"` + dquery := `DELETE FROM "` + c.config.MigrationsTable + `" WHERE version = ?` + iter := c.session.Query(squery).Iter() + var previous int + for iter.Scan(&previous) { + if err := c.session.Query(dquery, previous).Exec(); err != nil { + return &database.Error{OrigErr: err, Query: []byte(dquery)} + } + } + if err := iter.Close(); err != nil { + return &database.Error{OrigErr: err, Query: []byte(squery)} } - if version >= 0 { - query = `INSERT INTO "` + c.config.MigrationsTable + `" (version, dirty) VALUES (?, ?)` + + // Also re-write the schema version for nil dirty versions to prevent + // empty schema version for failed down migration on the first migration + // See: https://github.com/golang-migrate/migrate/issues/330 + if version >= 0 || (version == database.NilVersion && dirty) { + query := `INSERT INTO "` + c.config.MigrationsTable + `" (version, dirty) VALUES (?, ?)` if err := c.session.Query(query, version, dirty).Exec(); err != nil { return &database.Error{OrigErr: err, Query: []byte(query)} } @@ -227,13 +294,29 @@ func (c *Cassandra) Drop() error { return err } } - // Re-create the version table - return c.ensureVersionTable() + + return nil } -// Ensure version table exists -func (c *Cassandra) ensureVersionTable() error { - err := c.session.Query(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (version bigint, dirty boolean, PRIMARY KEY(version))", c.config.MigrationsTable)).Exec() +// ensureVersionTable checks if versions table exists and, if not, creates it. +// Note that this function locks the database, which deviates from the usual +// convention of "caller locks" in the Cassandra type. +func (c *Cassandra) ensureVersionTable() (err error) { + if err = c.Lock(); err != nil { + return err + } + + defer func() { + if e := c.Unlock(); e != nil { + if err == nil { + err = e + } else { + err = multierror.Append(err, e) + } + } + }() + + err = c.session.Query(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (version bigint, dirty boolean, PRIMARY KEY(version))", c.config.MigrationsTable)).Exec() if err != nil { return err } diff --git a/database/cassandra/cassandra_test.go b/database/cassandra/cassandra_test.go index 30107fa72..2e0c40283 100644 --- a/database/cassandra/cassandra_test.go +++ b/database/cassandra/cassandra_test.go @@ -1,30 +1,49 @@ package cassandra import ( + "context" "fmt" + "github.com/golang-migrate/migrate/v4" "strconv" "testing" +) +import ( + "github.com/dhui/dktest" "github.com/gocql/gocql" +) +import ( dt "github.com/golang-migrate/migrate/v4/database/testing" - mt "github.com/golang-migrate/migrate/v4/testing" + "github.com/golang-migrate/migrate/v4/dktesting" + _ "github.com/golang-migrate/migrate/v4/source/file" ) -var versions = []mt.Version{ - {Image: "cassandra:3.0.10"}, - {Image: "cassandra:3.0"}, -} +var ( + opts = dktest.Options{PortRequired: true, ReadyFunc: isReady} + // Supported versions: http://cassandra.apache.org/download/ + // Although Cassandra 2.x is supported by the Apache Foundation, + // the migrate db driver only supports Cassandra 3.x since it uses + // the system_schema keyspace. + specs = []dktesting.ContainerSpec{ + {ImageName: "cassandra:3.0", Options: opts}, + {ImageName: "cassandra:3.11", Options: opts}, + } +) -func isReady(i mt.Instance) bool { +func isReady(ctx context.Context, c dktest.ContainerInfo) bool { // Cassandra exposes 5 ports (7000, 7001, 7199, 9042 & 9160) - // We only need the port bound to 9042, but we can only access to the first one - // through 'i.Port()' (which calls DockerContainer.firstPortMapping()) - // So we need to get port mapping to retrieve correct port number bound to 9042 - portMap := i.NetworkSettings().Ports - port, _ := strconv.Atoi(portMap["9042/tcp"][0].HostPort) + // We only need the port bound to 9042 + ip, portStr, err := c.Port(9042) + if err != nil { + return false + } + port, err := strconv.Atoi(portStr) + if err != nil { + return false + } - cluster := gocql.NewCluster(i.Host()) + cluster := gocql.NewCluster(ip) cluster.Port = port cluster.Consistency = gocql.All p, err := cluster.CreateSession() @@ -40,17 +59,48 @@ func isReady(i mt.Instance) bool { } func Test(t *testing.T) { - mt.ParallelTest(t, versions, isReady, - func(t *testing.T, i mt.Instance) { - p := &Cassandra{} - portMap := i.NetworkSettings().Ports - port, _ := strconv.Atoi(portMap["9042/tcp"][0].HostPort) - addr := fmt.Sprintf("cassandra://%v:%v/testks", i.Host(), port) - d, err := p.Open(addr) - if err != nil { - t.Fatalf("%v", err) + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(9042) + if err != nil { + t.Fatal("Unable to get mapped port:", err) + } + addr := fmt.Sprintf("cassandra://%v:%v/testks", ip, port) + p := &Cassandra{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) } - defer d.Close() - dt.Test(t, d, []byte("SELECT table_name from system_schema.tables")) - }) + }() + dt.Test(t, d, []byte("SELECT table_name from system_schema.tables")) + }) +} + +func TestMigrate(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(9042) + if err != nil { + t.Fatal("Unable to get mapped port:", err) + } + addr := fmt.Sprintf("cassandra://%v:%v/testks", ip, port) + p := &Cassandra{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + + m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "testks", d) + if err != nil { + t.Fatal(err) + } + dt.TestMigrate(t, m) + }) } diff --git a/database/cassandra/examples/migrations/1_simple_select.down.sql b/database/cassandra/examples/migrations/1_simple_select.down.sql new file mode 100644 index 000000000..29787f084 --- /dev/null +++ b/database/cassandra/examples/migrations/1_simple_select.down.sql @@ -0,0 +1 @@ +SELECT table_name from system_schema.tables \ No newline at end of file diff --git a/database/cassandra/examples/migrations/1_simple_select.up.sql b/database/cassandra/examples/migrations/1_simple_select.up.sql new file mode 100644 index 000000000..29787f084 --- /dev/null +++ b/database/cassandra/examples/migrations/1_simple_select.up.sql @@ -0,0 +1 @@ +SELECT table_name from system_schema.tables \ No newline at end of file diff --git a/database/clickhouse/README.md b/database/clickhouse/README.md index b1ebe8817..359b9b70e 100644 --- a/database/clickhouse/README.md +++ b/database/clickhouse/README.md @@ -5,9 +5,11 @@ | URL Query | Description | |------------|-------------| | `x-migrations-table`| Name of the migrations table | +| `x-migrations-table-engine`| Engine to use for the migrations table, defaults to TinyLog | +| `x-cluster-name` | Name of cluster for creating `schema_migrations` table cluster wide | | `database` | The name of the database to connect to | | `username` | The user to sign in as | -| `password` | The user's password | +| `password` | The user's password | | `host` | The host to connect to. | | `port` | The port to bind to. | | `x-multi-statement` | false | Enable multiple statements to be ran in a single migration (See note below) | @@ -16,4 +18,8 @@ * The Clickhouse driver does not natively support executing multipe statements in a single query. To allow for multiple statements in a single migration, you can use the `x-multi-statement` param. There are two important caveats: * This mode splits the migration text into separately-executed statements by a semi-colon `;`. Thus `x-multi-statement` cannot be used when a statement in the migration contains a string with a semi-colon. - * The queries are not executed in any sort of transaction/batch, meaning you are responsible for fixing partial migrations. \ No newline at end of file + * The queries are not executed in any sort of transaction/batch, meaning you are responsible for fixing partial migrations. +* Using the default TinyLog table engine for the schema_versions table prevents backing up the table if using the [clickhouse-backup](https://github.com/AlexAkulov/clickhouse-backup) tool. If backing up the database with make sure the migrations are run with `x-migrations-table-engine=MergeTree`. +* Clickhouse cluster mode is not officially supported, since it's not tested right now, but you can try enabling `schema_migrations` table replication by specifying a `x-cluster-name`: + * When `x-cluster-name` is specified, `x-migrations-table-engine` also should be specified. See the docs regarding [replicated table engines](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/replication/#table_engines-replication). + * When `x-cluster-name` is specified, only the `schema_migrations` table is replicated across the cluster. You still need to write your migrations so that the application tables are replicated within the cluster. diff --git a/database/clickhouse/clickhouse.go b/database/clickhouse/clickhouse.go index c94ba725f..8cb6a8864 100644 --- a/database/clickhouse/clickhouse.go +++ b/database/clickhouse/clickhouse.go @@ -6,21 +6,35 @@ import ( "io" "io/ioutil" "net/url" + "strconv" "strings" "time" + "go.uber.org/atomic" + "github.com/golang-migrate/migrate/v4" "github.com/golang-migrate/migrate/v4/database" + "github.com/golang-migrate/migrate/v4/database/multistmt" + "github.com/hashicorp/go-multierror" ) -var DefaultMigrationsTable = "schema_migrations" +var ( + multiStmtDelimiter = []byte(";") + + DefaultMigrationsTable = "schema_migrations" + DefaultMigrationsTableEngine = "TinyLog" + DefaultMultiStatementMaxSize = 10 * 1 << 20 // 10 MB -var ErrNilConfig = fmt.Errorf("no config") + ErrNilConfig = fmt.Errorf("no config") +) type Config struct { DatabaseName string + ClusterName string MigrationsTable string + MigrationsTableEngine string MultiStatementEnabled bool + MultiStatementMaxSize int } func init() { @@ -49,8 +63,9 @@ func WithInstance(conn *sql.DB, config *Config) (database.Driver, error) { } type ClickHouse struct { - conn *sql.DB - config *Config + conn *sql.DB + config *Config + isLocked atomic.Bool } func (ch *ClickHouse) Open(dsn string) (database.Driver, error) { @@ -65,12 +80,28 @@ func (ch *ClickHouse) Open(dsn string) (database.Driver, error) { return nil, err } + multiStatementMaxSize := DefaultMultiStatementMaxSize + if s := purl.Query().Get("x-multi-statement-max-size"); len(s) > 0 { + multiStatementMaxSize, err = strconv.Atoi(s) + if err != nil { + return nil, err + } + } + + migrationsTableEngine := DefaultMigrationsTableEngine + if s := purl.Query().Get("x-migrations-table-engine"); len(s) > 0 { + migrationsTableEngine = s + } + ch = &ClickHouse{ conn: conn, config: &Config{ MigrationsTable: purl.Query().Get("x-migrations-table"), + MigrationsTableEngine: migrationsTableEngine, DatabaseName: purl.Query().Get("database"), + ClusterName: purl.Query().Get("x-cluster-name"), MultiStatementEnabled: purl.Query().Get("x-multi-statement") == "true", + MultiStatementMaxSize: multiStatementMaxSize, }, } @@ -92,28 +123,39 @@ func (ch *ClickHouse) init() error { ch.config.MigrationsTable = DefaultMigrationsTable } + if ch.config.MultiStatementMaxSize <= 0 { + ch.config.MultiStatementMaxSize = DefaultMultiStatementMaxSize + } + + if len(ch.config.MigrationsTableEngine) == 0 { + ch.config.MigrationsTableEngine = DefaultMigrationsTableEngine + } + return ch.ensureVersionTable() } func (ch *ClickHouse) Run(r io.Reader) error { - migration, err := ioutil.ReadAll(r) - if err != nil { - return err - } - if ch.config.MultiStatementEnabled { - // split query by semi-colon - queries := strings.Split(string(migration), ";") - for _, q := range queries { - tq := strings.TrimSpace(q) + var err error + if e := multistmt.Parse(r, multiStmtDelimiter, ch.config.MultiStatementMaxSize, func(m []byte) bool { + tq := strings.TrimSpace(string(m)) if tq == "" { - continue + return true } - if _, err := ch.conn.Exec(string(q)); err != nil { - return database.Error{OrigErr: err, Err: "migration failed", Query: []byte(q)} + if _, e := ch.conn.Exec(string(m)); e != nil { + err = database.Error{OrigErr: e, Err: "migration failed", Query: m} + return false } + return true + }); e != nil { + return e } - return nil + return err + } + + migration, err := ioutil.ReadAll(r) + if err != nil { + return err } if _, err := ch.conn.Exec(string(migration)); err != nil { @@ -159,7 +201,24 @@ func (ch *ClickHouse) SetVersion(version int, dirty bool) error { return tx.Commit() } -func (ch *ClickHouse) ensureVersionTable() error { +// ensureVersionTable checks if versions table exists and, if not, creates it. +// Note that this function locks the database, which deviates from the usual +// convention of "caller locks" in the ClickHouse type. +func (ch *ClickHouse) ensureVersionTable() (err error) { + if err = ch.Lock(); err != nil { + return err + } + + defer func() { + if e := ch.Unlock(); e != nil { + if err == nil { + err = e + } else { + err = multierror.Append(err, e) + } + } + }() + var ( table string query = "SHOW TABLES FROM " + ch.config.DatabaseName + " LIKE '" + ch.config.MigrationsTable + "'" @@ -172,29 +231,47 @@ func (ch *ClickHouse) ensureVersionTable() error { } else { return nil } + // if not, create the empty migration table - query = ` - CREATE TABLE ` + ch.config.MigrationsTable + ` ( - version UInt32, - dirty UInt8, - sequence UInt64 - ) Engine=TinyLog - ` + if len(ch.config.ClusterName) > 0 { + query = fmt.Sprintf(` + CREATE TABLE %s ON CLUSTER %s ( + version Int64, + dirty UInt8, + sequence UInt64 + ) Engine=%s`, ch.config.MigrationsTable, ch.config.ClusterName, ch.config.MigrationsTableEngine) + } else { + query = fmt.Sprintf(` + CREATE TABLE %s ( + version Int64, + dirty UInt8, + sequence UInt64 + ) Engine=%s`, ch.config.MigrationsTable, ch.config.MigrationsTableEngine) + } + + if strings.HasSuffix(ch.config.MigrationsTableEngine, "Tree") { + query = fmt.Sprintf(`%s ORDER BY sequence`, query) + } + if _, err := ch.conn.Exec(query); err != nil { return &database.Error{OrigErr: err, Query: []byte(query)} } return nil } -func (ch *ClickHouse) Drop() error { - var ( - query = "SHOW TABLES FROM " + ch.config.DatabaseName - tables, err = ch.conn.Query(query) - ) +func (ch *ClickHouse) Drop() (err error) { + query := "SHOW TABLES FROM " + ch.config.DatabaseName + tables, err := ch.conn.Query(query) + if err != nil { return &database.Error{OrigErr: err, Query: []byte(query)} } - defer tables.Close() + defer func() { + if errClose := tables.Close(); errClose != nil { + err = multierror.Append(err, errClose) + } + }() + for tables.Next() { var table string if err := tables.Scan(&table); err != nil { @@ -207,9 +284,25 @@ func (ch *ClickHouse) Drop() error { return &database.Error{OrigErr: err, Query: []byte(query)} } } - return ch.ensureVersionTable() + if err := tables.Err(); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + return nil } -func (ch *ClickHouse) Lock() error { return nil } -func (ch *ClickHouse) Unlock() error { return nil } -func (ch *ClickHouse) Close() error { return ch.conn.Close() } +func (ch *ClickHouse) Lock() error { + if !ch.isLocked.CAS(false, true) { + return database.ErrLocked + } + + return nil +} +func (ch *ClickHouse) Unlock() error { + if !ch.isLocked.CAS(true, false) { + return database.ErrNotLocked + } + + return nil +} +func (ch *ClickHouse) Close() error { return ch.conn.Close() } diff --git a/database/clickhouse/clickhouse_test.go b/database/clickhouse/clickhouse_test.go new file mode 100644 index 000000000..694aa2a7b --- /dev/null +++ b/database/clickhouse/clickhouse_test.go @@ -0,0 +1,224 @@ +package clickhouse_test + +import ( + "context" + "database/sql" + sqldriver "database/sql/driver" + "fmt" + "log" + "testing" + + _ "github.com/ClickHouse/clickhouse-go" + "github.com/dhui/dktest" + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database/clickhouse" + dt "github.com/golang-migrate/migrate/v4/database/testing" + "github.com/golang-migrate/migrate/v4/dktesting" + _ "github.com/golang-migrate/migrate/v4/source/file" +) + +const defaultPort = 9000 + +var ( + tableEngines = []string{"TinyLog", "MergeTree"} + opts = dktest.Options{ + Env: map[string]string{"CLICKHOUSE_USER": "user", "CLICKHOUSE_PASSWORD": "password", "CLICKHOUSE_DB": "db"}, + PortRequired: true, ReadyFunc: isReady, + } + specs = []dktesting.ContainerSpec{ + {ImageName: "yandex/clickhouse-server:21.3", Options: opts}, + } +) + +func clickhouseConnectionString(host, port, engine string) string { + if engine != "" { + return fmt.Sprintf( + "clickhouse://%v:%v?username=user&password=password&database=db&x-multi-statement=true&x-migrations-table-engine=%v&debug=false", + host, port, engine) + } + + return fmt.Sprintf( + "clickhouse://%v:%v?username=user&password=password&database=db&x-multi-statement=true&debug=false", + host, port) +} + +func isReady(ctx context.Context, c dktest.ContainerInfo) bool { + ip, port, err := c.Port(defaultPort) + if err != nil { + return false + } + + db, err := sql.Open("clickhouse", clickhouseConnectionString(ip, port, "")) + + if err != nil { + log.Println("open error", err) + return false + } + defer func() { + if err := db.Close(); err != nil { + log.Println("close error:", err) + } + }() + + if err = db.PingContext(ctx); err != nil { + switch err { + case sqldriver.ErrBadConn: + return false + default: + fmt.Println(err) + } + return false + } + + return true +} + +func TestCases(t *testing.T) { + for _, engine := range tableEngines { + t.Run("Test_"+engine, func(t *testing.T) { testSimple(t, engine) }) + t.Run("Migrate_"+engine, func(t *testing.T) { testMigrate(t, engine) }) + t.Run("Version_"+engine, func(t *testing.T) { testVersion(t, engine) }) + t.Run("Drop_"+engine, func(t *testing.T) { testDrop(t, engine) }) + } + t.Run("WithInstanceDefaultConfigValues", func(t *testing.T) { testSimpleWithInstanceDefaultConfigValues(t) }) +} + +func testSimple(t *testing.T, engine string) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } + + addr := clickhouseConnectionString(ip, port, engine) + p := &clickhouse.ClickHouse{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + + dt.Test(t, d, []byte("SELECT 1")) + }) +} + +func testSimpleWithInstanceDefaultConfigValues(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } + + addr := clickhouseConnectionString(ip, port, "") + conn, err := sql.Open("clickhouse", addr) + if err != nil { + t.Fatal(err) + } + d, err := clickhouse.WithInstance(conn, &clickhouse.Config{}) + if err != nil { + _ = conn.Close() + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + + dt.Test(t, d, []byte("SELECT 1")) + }) +} + +func testMigrate(t *testing.T, engine string) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } + + addr := clickhouseConnectionString(ip, port, engine) + p := &clickhouse.ClickHouse{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "db", d) + + if err != nil { + t.Fatal(err) + } + dt.TestMigrate(t, m) + }) +} + +func testVersion(t *testing.T, engine string) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + expectedVersion := 1 + + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } + + addr := clickhouseConnectionString(ip, port, engine) + p := &clickhouse.ClickHouse{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + + err = d.SetVersion(expectedVersion, false) + if err != nil { + t.Fatal(err) + } + + version, _, err := d.Version() + if err != nil { + t.Fatal(err) + } + + if version != expectedVersion { + t.Fatal("Version mismatch") + } + }) +} + +func testDrop(t *testing.T, engine string) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } + + addr := clickhouseConnectionString(ip, port, engine) + p := &clickhouse.ClickHouse{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + + err = d.Drop() + if err != nil { + t.Fatal(err) + } + }) +} diff --git a/database/cockroachdb/TUTORIAL.md b/database/cockroachdb/TUTORIAL.md new file mode 100644 index 000000000..c6604165e --- /dev/null +++ b/database/cockroachdb/TUTORIAL.md @@ -0,0 +1,142 @@ +# CockroachDB tutorial for beginners (insecure cluster) + +## Create/configure database + +First, let's start a local cluster - follow step 1. and 2. from [the docs](https://www.cockroachlabs.com/docs/stable/start-a-local-cluster.html#step-1-start-the-first-node). + +Once you have it, create a database. Here I am going to create a database called `example`. +Our user here is `cockroach`. We are not going to use a password, since it's not supported for insecure cluster. +``` +cockroach sql --insecure --host=localhost:26257 +``` +``` +CREATE DATABASE example; +CREATE USER IF NOT EXISTS cockroach; +GRANT ALL ON DATABASE example TO cockroach; +``` + +When using Migrate CLI we need to pass to database URL. Let's export it to a variable for convienience: +``` +export COCKROACHDB_URL='cockroachdb://cockroach:@localhost:26257/example?sslmode=disable' +``` +`sslmode=disable` means that the connection with our database will not be encrypted. This is needed to connect to an insecure node. + +**NOTE:** Do not use COCKROACH_URL as a variable name here, it's already in use for discrete parameters and you may run into connection problems. For more info check out [docs](https://www.cockroachlabs.com/docs/stable/connection-parameters.html#connect-using-discrete-parameters). + +You can find further description of database URLs [here](README.md#database-urls). + +## Create migrations +Let's create a table called `users`: +``` +migrate create -ext sql -dir db/migrations -seq create_users_table +``` +If there were no errors, we should have two files available under `db/migrations` folder: +- 000001_create_users_table.down.sql +- 000001_create_users_table.up.sql + +Note the `sql` extension that we provided. + +In the `.up.sql` file let's create the table: +``` +CREATE TABLE IF NOT EXISTS example.users +( + user_id INT PRIMARY KEY, + username VARCHAR (50) UNIQUE NOT NULL, + password VARCHAR (50) NOT NULL, + email VARCHAR (300) UNIQUE NOT NULL +); +``` +And in the `.down.sql` let's delete it: +``` +DROP TABLE IF EXISTS example.users; +``` +By adding `IF EXISTS/IF NOT EXISTS` we are making migrations idempotent - you can read more about idempotency in [getting started](GETTING_STARTED.md#create-migrations) + +## Run migrations +``` +migrate -database ${COCKROACHDB_URL} -path db/migrations up +``` +Let's check if the table was created properly by running `cockroach sql --insecure --host=localhost:26257 -e "show columns from example.users;"`. +The output you are supposed to see: +``` + column_name | data_type | is_nullable | column_default | generation_expression | indices | is_hidden ++-------------+--------------+-------------+----------------+-----------------------+----------------------------------------------+-----------+ + user_id | INT8 | false | NULL | | {primary,users_username_key,users_email_key} | false + username | VARCHAR(50) | false | NULL | | {users_username_key} | false + password | VARCHAR(50) | false | NULL | | {} | false + email | VARCHAR(300) | false | NULL | | {users_email_key} | false +(4 rows) +``` +Now let's check if running reverse migration also works: +``` +migrate -database ${COCKROACHDB_URL} -path db/migrations down +``` +Make sure to check if your database changed as expected in this case as well. + +## Database transactions + +To show database transactions usage, let's create another set of migrations by running: +``` +migrate create -ext sql -dir db/migrations -seq add_mood_to_users +``` +Again, it should create for us two migrations files: +- 000002_add_mood_to_users.down.sql +- 000002_add_mood_to_users.up.sql + +In Cockroach, when we want our queries to be done in a transaction, we need to wrap it with `BEGIN` and `COMMIT` commands, similar to PostgreSQL. +In our example, we are going to add a column to our database that can only accept enumerable values or NULL. +Migration up: +``` +BEGIN; + +ALTER TABLE example.users ADD COLUMN mood STRING; +ALTER TABLE example.users ADD CONSTRAINT check_mood CHECK (mood IN ('happy', 'sad', 'neutral')); + +COMMIT; +``` +Migration down: +``` +ALTER TABLE example.users DROP COLUMN mood; +``` + +Now we can run our new migration and check the database: +``` +migrate -database ${COCKROACHDB_URL} -path db/migrations up +cockroach sql --insecure --host=localhost:26257 -e "show columns from example.users;" +``` +Expected output: +``` + column_name | data_type | is_nullable | column_default | generation_expression | indices | is_hidden ++-------------+--------------+-------------+----------------+-----------------------+----------------------------------------------+-----------+ + user_id | INT8 | false | NULL | | {primary,users_username_key,users_email_key} | false + username | VARCHAR(50) | false | NULL | | {users_username_key} | false + password | VARCHAR(50) | false | NULL | | {} | false + email | VARCHAR(300) | false | NULL | | {users_email_key} | false + mood | STRING | true | NULL | | {} | false +(5 rows) +``` + +## Optional: Run migrations within your Go app +Here is a very simple app running migrations for the above configuration: +``` +import ( + "log" + + "github.com/golang-migrate/migrate/v4" + _ "github.com/golang-migrate/migrate/v4/database/cockroachdb" + _ "github.com/golang-migrate/migrate/v4/source/file" +) + +func main() { + m, err := migrate.New( + "file://db/migrations", + "cockroachdb://cockroach:@localhost:26257/example?sslmode=disable") + if err != nil { + log.Fatal(err) + } + if err := m.Up(); err != nil { + log.Fatal(err) + } +} +``` +You can find details [here](README.md#use-in-your-go-project) \ No newline at end of file diff --git a/database/cockroachdb/cockroachdb.go b/database/cockroachdb/cockroachdb.go index df32db0d8..b345cbc63 100644 --- a/database/cockroachdb/cockroachdb.go +++ b/database/cockroachdb/cockroachdb.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "go.uber.org/atomic" "io" "io/ioutil" nurl "net/url" @@ -12,7 +13,8 @@ import ( ) import ( - "github.com/cockroachdb/cockroach-go/crdb" + "github.com/cockroachdb/cockroach-go/v2/crdb" + "github.com/hashicorp/go-multierror" "github.com/lib/pq" ) @@ -45,7 +47,7 @@ type Config struct { type CockroachDb struct { db *sql.DB - isLocked bool + isLocked atomic.Bool // Open and WithInstance need to guarantee that config is never nil config *Config @@ -60,17 +62,19 @@ func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { return nil, err } - query := `SELECT current_database()` - var databaseName string - if err := instance.QueryRow(query).Scan(&databaseName); err != nil { - return nil, &database.Error{OrigErr: err, Query: []byte(query)} - } + if config.DatabaseName == "" { + query := `SELECT current_database()` + var databaseName string + if err := instance.QueryRow(query).Scan(&databaseName); err != nil { + return nil, &database.Error{OrigErr: err, Query: []byte(query)} + } - if len(databaseName) == 0 { - return nil, ErrNoDatabaseName - } + if len(databaseName) == 0 { + return nil, ErrNoDatabaseName + } - config.DatabaseName = databaseName + config.DatabaseName = databaseName + } if len(config.MigrationsTable) == 0 { config.MigrationsTable = DefaultMigrationsTable @@ -85,11 +89,12 @@ func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { config: config, } - if err := px.ensureVersionTable(); err != nil { + // ensureVersionTable is a locking operation, so we need to ensureLockTable before we ensureVersionTable. + if err := px.ensureLockTable(); err != nil { return nil, err } - if err := px.ensureLockTable(); err != nil { + if err := px.ensureVersionTable(); err != nil { return nil, err } @@ -148,67 +153,67 @@ func (c *CockroachDb) Close() error { // Locking is done manually with a separate lock table. Implementing advisory locks in CRDB is being discussed // See: https://github.com/cockroachdb/cockroach/issues/13546 func (c *CockroachDb) Lock() error { - err := crdb.ExecuteTx(context.Background(), c.db, nil, func(tx *sql.Tx) error { - aid, err := database.GenerateAdvisoryLockId(c.config.DatabaseName) - if err != nil { - return err - } - - query := "SELECT * FROM " + c.config.LockTable + " WHERE lock_id = $1" - rows, err := tx.Query(query, aid) - if err != nil { - return database.Error{OrigErr: err, Err: "failed to fetch migration lock", Query: []byte(query)} - } - defer rows.Close() + return database.CasRestoreOnErr(&c.isLocked, false, true, database.ErrLocked, func() (err error) { + return crdb.ExecuteTx(context.Background(), c.db, nil, func(tx *sql.Tx) (err error) { + aid, err := database.GenerateAdvisoryLockId(c.config.DatabaseName) + if err != nil { + return err + } - // If row exists at all, lock is present - locked := rows.Next() - if locked && !c.config.ForceLock { - return database.ErrLocked - } + query := "SELECT * FROM " + c.config.LockTable + " WHERE lock_id = $1" + rows, err := tx.Query(query, aid) + if err != nil { + return database.Error{OrigErr: err, Err: "failed to fetch migration lock", Query: []byte(query)} + } + defer func() { + if errClose := rows.Close(); errClose != nil { + err = multierror.Append(err, errClose) + } + }() + + // If row exists at all, lock is present + locked := rows.Next() + if locked && !c.config.ForceLock { + return database.ErrLocked + } - query = "INSERT INTO " + c.config.LockTable + " (lock_id) VALUES ($1)" - if _, err := tx.Exec(query, aid); err != nil { - return database.Error{OrigErr: err, Err: "failed to set migration lock", Query: []byte(query)} - } + query = "INSERT INTO " + c.config.LockTable + " (lock_id) VALUES ($1)" + if _, err := tx.Exec(query, aid); err != nil { + return database.Error{OrigErr: err, Err: "failed to set migration lock", Query: []byte(query)} + } - return nil + return nil + }) }) - - if err != nil { - return err - } else { - c.isLocked = true - return nil - } } // Locking is done manually with a separate lock table. Implementing advisory locks in CRDB is being discussed // See: https://github.com/cockroachdb/cockroach/issues/13546 func (c *CockroachDb) Unlock() error { - aid, err := database.GenerateAdvisoryLockId(c.config.DatabaseName) - if err != nil { - return err - } + return database.CasRestoreOnErr(&c.isLocked, true, false, database.ErrNotLocked, func() (err error) { + aid, err := database.GenerateAdvisoryLockId(c.config.DatabaseName) + if err != nil { + return err + } - // In the event of an implementation (non-migration) error, it is possible for the lock to not be released. Until - // a better locking mechanism is added, a manual purging of the lock table may be required in such circumstances - query := "DELETE FROM " + c.config.LockTable + " WHERE lock_id = $1" - if _, err := c.db.Exec(query, aid); err != nil { - if e, ok := err.(*pq.Error); ok { - // 42P01 is "UndefinedTableError" in CockroachDB - // https://github.com/cockroachdb/cockroach/blob/master/pkg/sql/pgwire/pgerror/codes.go - if e.Code == "42P01" { - // On drops, the lock table is fully removed; This is fine, and is a valid "unlocked" state for the schema - c.isLocked = false - return nil + // In the event of an implementation (non-migration) error, it is possible for the lock to not be released. Until + // a better locking mechanism is added, a manual purging of the lock table may be required in such circumstances + query := "DELETE FROM " + c.config.LockTable + " WHERE lock_id = $1" + if _, err := c.db.Exec(query, aid); err != nil { + if e, ok := err.(*pq.Error); ok { + // 42P01 is "UndefinedTableError" in CockroachDB + // https://github.com/cockroachdb/cockroach/blob/master/pkg/sql/pgwire/pgerror/codes.go + if e.Code == "42P01" { + // On drops, the lock table is fully removed; This is fine, and is a valid "unlocked" state for the schema + return nil + } } + + return database.Error{OrigErr: err, Err: "failed to release migration lock", Query: []byte(query)} } - return database.Error{OrigErr: err, Err: "failed to release migration lock", Query: []byte(query)} - } - c.isLocked = false - return nil + return nil + }) } func (c *CockroachDb) Run(migration io.Reader) error { @@ -232,7 +237,10 @@ func (c *CockroachDb) SetVersion(version int, dirty bool) error { return err } - if version >= 0 { + // Also re-write the schema version for nil dirty versions to prevent + // empty schema version for failed down migration on the first migration + // See: https://github.com/golang-migrate/migrate/issues/330 + if version >= 0 || (version == database.NilVersion && dirty) { if _, err := tx.Exec(`INSERT INTO "`+c.config.MigrationsTable+`" (version, dirty) VALUES ($1, $2)`, version, dirty); err != nil { return err } @@ -265,14 +273,18 @@ func (c *CockroachDb) Version() (version int, dirty bool, err error) { } } -func (c *CockroachDb) Drop() error { +func (c *CockroachDb) Drop() (err error) { // select all tables in current schema query := `SELECT table_name FROM information_schema.tables WHERE table_schema=(SELECT current_schema())` tables, err := c.db.Query(query) if err != nil { return &database.Error{OrigErr: err, Query: []byte(query)} } - defer tables.Close() + defer func() { + if errClose := tables.Close(); errClose != nil { + err = multierror.Append(err, errClose) + } + }() // delete one table after another tableNames := make([]string, 0) @@ -285,6 +297,9 @@ func (c *CockroachDb) Drop() error { tableNames = append(tableNames, tableName) } } + if err := tables.Err(); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } if len(tableNames) > 0 { // delete one by one ... @@ -294,15 +309,29 @@ func (c *CockroachDb) Drop() error { return &database.Error{OrigErr: err, Query: []byte(query)} } } - if err := c.ensureVersionTable(); err != nil { - return err - } } return nil } -func (c *CockroachDb) ensureVersionTable() error { +// ensureVersionTable checks if versions table exists and, if not, creates it. +// Note that this function locks the database, which deviates from the usual +// convention of "caller locks" in the CockroachDb type. +func (c *CockroachDb) ensureVersionTable() (err error) { + if err = c.Lock(); err != nil { + return err + } + + defer func() { + if e := c.Unlock(); e != nil { + if err == nil { + err = e + } else { + err = multierror.Append(err, e) + } + } + }() + // check if migration table exists var count int query := `SELECT COUNT(1) FROM information_schema.tables WHERE table_name = $1 AND table_schema = (SELECT current_schema()) LIMIT 1` diff --git a/database/cockroachdb/cockroachdb_test.go b/database/cockroachdb/cockroachdb_test.go index df4142278..d00e27503 100644 --- a/database/cockroachdb/cockroachdb_test.go +++ b/database/cockroachdb/cockroachdb_test.go @@ -3,89 +3,172 @@ package cockroachdb // error codes https://github.com/lib/pq/blob/master/error.go import ( - //"bytes" + "context" "database/sql" "fmt" - "io" + "github.com/golang-migrate/migrate/v4" + "log" + "strings" "testing" +) + +import ( + "github.com/dhui/dktest" + _ "github.com/lib/pq" +) - "bytes" +import ( dt "github.com/golang-migrate/migrate/v4/database/testing" - mt "github.com/golang-migrate/migrate/v4/testing" - "github.com/lib/pq" + "github.com/golang-migrate/migrate/v4/dktesting" + _ "github.com/golang-migrate/migrate/v4/source/file" ) -var versions = []mt.Version{ - {Image: "cockroachdb/cockroach:v1.0.2", Cmd: []string{"start", "--insecure"}}, -} +const defaultPort = 26257 -func isReady(i mt.Instance) bool { - db, err := sql.Open("postgres", fmt.Sprintf("postgres://root@%v:%v?sslmode=disable", i.Host(), i.PortFor(26257))) +var ( + opts = dktest.Options{Cmd: []string{"start", "--insecure"}, PortRequired: true, ReadyFunc: isReady} + // Released versions: https://www.cockroachlabs.com/docs/releases/ + specs = []dktesting.ContainerSpec{ + {ImageName: "cockroachdb/cockroach:v1.0.7", Options: opts}, + {ImageName: "cockroachdb/cockroach:v1.1.9", Options: opts}, + {ImageName: "cockroachdb/cockroach:v2.0.7", Options: opts}, + {ImageName: "cockroachdb/cockroach:v2.1.3", Options: opts}, + } +) + +func isReady(ctx context.Context, c dktest.ContainerInfo) bool { + ip, port, err := c.Port(defaultPort) if err != nil { + log.Println("port error:", err) return false } - defer db.Close() - err = db.Ping() - if err == io.EOF { - _, err = db.Exec("CREATE DATABASE migrate") - return err == nil - } else if e, ok := err.(*pq.Error); ok { - if e.Code.Name() == "cannot_connect_now" { - return false - } + + db, err := sql.Open("postgres", fmt.Sprintf("postgres://root@%v:%v?sslmode=disable", ip, port)) + if err != nil { + log.Println("open error:", err) + return false } + if err := db.PingContext(ctx); err != nil { + log.Println("ping error:", err) + return false + } + if err := db.Close(); err != nil { + log.Println("close error:", err) + } + return true +} - _, err = db.Exec("CREATE DATABASE migrate") - return err == nil +func createDB(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } - return true + db, err := sql.Open("postgres", fmt.Sprintf("postgres://root@%v:%v?sslmode=disable", ip, port)) + if err != nil { + t.Fatal(err) + } + if err = db.Ping(); err != nil { + t.Fatal(err) + } + defer func() { + if err := db.Close(); err != nil { + t.Error(err) + } + }() + + if _, err = db.Exec("CREATE DATABASE migrate"); err != nil { + t.Fatal(err) + } } func Test(t *testing.T) { - mt.ParallelTest(t, versions, isReady, - func(t *testing.T, i mt.Instance) { - c := &CockroachDb{} - addr := fmt.Sprintf("cockroach://root@%v:%v/migrate?sslmode=disable", i.Host(), i.PortFor(26257)) - d, err := c.Open(addr) - if err != nil { - t.Fatalf("%v", err) - } - dt.Test(t, d, []byte("SELECT 1")) - }) + dktesting.ParallelTest(t, specs, func(t *testing.T, ci dktest.ContainerInfo) { + createDB(t, ci) + + ip, port, err := ci.Port(26257) + if err != nil { + t.Fatal(err) + } + + addr := fmt.Sprintf("cockroach://root@%v:%v/migrate?sslmode=disable", ip, port) + c := &CockroachDb{} + d, err := c.Open(addr) + if err != nil { + t.Fatal(err) + } + dt.Test(t, d, []byte("SELECT 1")) + }) +} + +func TestMigrate(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, ci dktest.ContainerInfo) { + createDB(t, ci) + + ip, port, err := ci.Port(26257) + if err != nil { + t.Fatal(err) + } + + addr := fmt.Sprintf("cockroach://root@%v:%v/migrate?sslmode=disable", ip, port) + c := &CockroachDb{} + d, err := c.Open(addr) + if err != nil { + t.Fatal(err) + } + + m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "migrate", d) + if err != nil { + t.Fatal(err) + } + dt.TestMigrate(t, m) + }) } func TestMultiStatement(t *testing.T) { - mt.ParallelTest(t, versions, isReady, - func(t *testing.T, i mt.Instance) { - c := &CockroachDb{} - addr := fmt.Sprintf("cockroach://root@%v:%v/migrate?sslmode=disable", i.Host(), i.Port()) - d, err := c.Open(addr) - if err != nil { - t.Fatalf("%v", err) - } - if err := d.Run(bytes.NewReader([]byte("CREATE TABLE foo (foo text); CREATE TABLE bar (bar text);"))); err != nil { - t.Fatalf("expected err to be nil, got %v", err) - } - - // make sure second table exists - var exists bool - if err := d.(*CockroachDb).db.QueryRow("SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'bar' AND table_schema = (SELECT current_schema()))").Scan(&exists); err != nil { - t.Fatal(err) - } - if !exists { - t.Fatalf("expected table bar to exist") - } - }) + dktesting.ParallelTest(t, specs, func(t *testing.T, ci dktest.ContainerInfo) { + createDB(t, ci) + + ip, port, err := ci.Port(26257) + if err != nil { + t.Fatal(err) + } + + addr := fmt.Sprintf("cockroach://root@%v:%v/migrate?sslmode=disable", ip, port) + c := &CockroachDb{} + d, err := c.Open(addr) + if err != nil { + t.Fatal(err) + } + if err := d.Run(strings.NewReader("CREATE TABLE foo (foo text); CREATE TABLE bar (bar text);")); err != nil { + t.Fatalf("expected err to be nil, got %v", err) + } + + // make sure second table exists + var exists bool + if err := d.(*CockroachDb).db.QueryRow("SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'bar' AND table_schema = (SELECT current_schema()))").Scan(&exists); err != nil { + t.Fatal(err) + } + if !exists { + t.Fatalf("expected table bar to exist") + } + }) } func TestFilterCustomQuery(t *testing.T) { - mt.ParallelTest(t, versions, isReady, - func(t *testing.T, i mt.Instance) { - c := &CockroachDb{} - addr := fmt.Sprintf("cockroach://root@%v:%v/migrate?sslmode=disable&x-custom=foobar", i.Host(), i.PortFor(26257)) - _, err := c.Open(addr) - if err != nil { - t.Fatalf("%v", err) - } - }) + dktesting.ParallelTest(t, specs, func(t *testing.T, ci dktest.ContainerInfo) { + createDB(t, ci) + + ip, port, err := ci.Port(26257) + if err != nil { + t.Fatal(err) + } + + addr := fmt.Sprintf("cockroach://root@%v:%v/migrate?sslmode=disable&x-custom=foobar", ip, port) + c := &CockroachDb{} + _, err = c.Open(addr) + if err != nil { + t.Fatal(err) + } + }) } diff --git a/database/driver.go b/database/driver.go index fa914c5d0..fa80f455c 100644 --- a/database/driver.go +++ b/database/driver.go @@ -7,12 +7,14 @@ package database import ( "fmt" "io" - nurl "net/url" "sync" + + iurl "github.com/golang-migrate/migrate/v4/internal/url" ) var ( - ErrLocked = fmt.Errorf("can't acquire lock") + ErrLocked = fmt.Errorf("can't acquire lock") + ErrNotLocked = fmt.Errorf("can't unlock, as not currently locked") ) const NilVersion int = -1 @@ -32,7 +34,7 @@ var drivers = make(map[string]Driver) // All other functions are tested by tests in database/testing. // Saves you some time and makes sure all database drivers behave the same way. // 5. Call Register in init(). -// 6. Create a migrate/cli/build_.go file +// 6. Create a internal/cli/build_.go file // 7. Add driver name in 'DATABASE' variable in Makefile // // Guidelines: @@ -60,7 +62,7 @@ type Driver interface { // all migrations have been run. Unlock() error - // Run applies a migration to the database. migration is garantueed to be not nil. + // Run applies a migration to the database. migration is guaranteed to be not nil. Run(migration io.Reader) error // SetVersion saves version and dirty state. @@ -74,26 +76,23 @@ type Driver interface { Version() (version int, dirty bool, err error) // Drop deletes everything in the database. + // Note that this is a breaking action, a new call to Open() is necessary to + // ensure subsequent calls work as expected. Drop() error } // Open returns a new driver instance. func Open(url string) (Driver, error) { - u, err := nurl.Parse(url) + scheme, err := iurl.SchemeFromURL(url) if err != nil { - return nil, fmt.Errorf("Unable to parse URL. Did you escape all reserved URL characters? "+ - "See: https://github.com/golang-migrate/migrate#database-urls Error: %v", err) - } - - if u.Scheme == "" { - return nil, fmt.Errorf("database driver: invalid URL scheme") + return nil, err } driversMu.RLock() - d, ok := drivers[u.Scheme] + d, ok := drivers[scheme] driversMu.RUnlock() if !ok { - return nil, fmt.Errorf("database driver: unknown driver %v (forgotten import?)", u.Scheme) + return nil, fmt.Errorf("database driver: unknown driver %v (forgotten import?)", scheme) } return d.Open(url) diff --git a/database/driver_test.go b/database/driver_test.go index c0a29304f..7880f3208 100644 --- a/database/driver_test.go +++ b/database/driver_test.go @@ -1,8 +1,115 @@ package database +import ( + "io" + "testing" +) + func ExampleDriver() { // see database/stub for an example // database/stub/stub.go has the driver implementation // database/stub/stub_test.go runs database/testing/test.go:Test } + +// Using database/stub here is not possible as it +// results in an import cycle. +type mockDriver struct { + url string +} + +func (m *mockDriver) Open(url string) (Driver, error) { + return &mockDriver{ + url: url, + }, nil +} + +func (m *mockDriver) Close() error { + return nil +} + +func (m *mockDriver) Lock() error { + return nil +} + +func (m *mockDriver) Unlock() error { + return nil +} + +func (m *mockDriver) Run(migration io.Reader) error { + return nil +} + +func (m *mockDriver) SetVersion(version int, dirty bool) error { + return nil +} + +func (m *mockDriver) Version() (version int, dirty bool, err error) { + return 0, false, nil +} + +func (m *mockDriver) Drop() error { + return nil +} + +func TestRegisterTwice(t *testing.T) { + Register("mock", &mockDriver{}) + + var err interface{} + func() { + defer func() { + err = recover() + }() + Register("mock", &mockDriver{}) + }() + + if err == nil { + t.Fatal("expected a panic when calling Register twice") + } +} + +func TestOpen(t *testing.T) { + // Make sure the driver is registered. + // But if the previous test already registered it just ignore the panic. + // If we don't do this it will be impossible to run this test standalone. + func() { + defer func() { + _ = recover() + }() + Register("mock", &mockDriver{}) + }() + + cases := []struct { + url string + err bool + }{ + { + "mock://user:pass@tcp(host:1337)/db", + false, + }, + { + "unknown://bla", + true, + }, + } + + for _, c := range cases { + t.Run(c.url, func(t *testing.T) { + d, err := Open(c.url) + + if err == nil { + if c.err { + t.Fatal("expected an error for an unknown driver") + } else { + if md, ok := d.(*mockDriver); !ok { + t.Fatalf("expected *mockDriver got %T", d) + } else if md.url != c.url { + t.Fatalf("expected %q got %q", c.url, md.url) + } + } + } else if !c.err { + t.Fatalf("did not expect %q", err) + } + }) + } +} diff --git a/database/firebird/README.md b/database/firebird/README.md new file mode 100644 index 000000000..bdfef8aa9 --- /dev/null +++ b/database/firebird/README.md @@ -0,0 +1,12 @@ +# firebird + +`firebirdsql://user:password@servername[:port_number]/database_name_or_file[?params1=value1[¶m2=value2]...]` + +| URL Query | WithInstance Config | Description | +|------------|---------------------|-------------| +| `x-migrations-table` | `MigrationsTable` | Name of the migrations table | +| `auth_plugin_name` | | Authentication plugin name. Srp256/Srp/Legacy_Auth are available. (default is Srp) | +| `column_name_to_lower` | | Force column name to lower. (default is false) | +| `role` | | Role name | +| `tzname` | | Time Zone name. (For Firebird 4.0+) | +| `wire_crypt` | | Enable wire data encryption or not. For Firebird 3.0+ (default is true) | diff --git a/database/firebird/examples/migrations/1085649617_create_users_table.down.sql b/database/firebird/examples/migrations/1085649617_create_users_table.down.sql new file mode 100644 index 000000000..cc1f647d2 --- /dev/null +++ b/database/firebird/examples/migrations/1085649617_create_users_table.down.sql @@ -0,0 +1 @@ +DROP TABLE users; diff --git a/database/firebird/examples/migrations/1085649617_create_users_table.up.sql b/database/firebird/examples/migrations/1085649617_create_users_table.up.sql new file mode 100644 index 000000000..92897dcab --- /dev/null +++ b/database/firebird/examples/migrations/1085649617_create_users_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE users ( + user_id integer unique, + name varchar(40), + email varchar(40) +); diff --git a/database/firebird/examples/migrations/1185749658_add_city_to_users.down.sql b/database/firebird/examples/migrations/1185749658_add_city_to_users.down.sql new file mode 100644 index 000000000..83aa5c875 --- /dev/null +++ b/database/firebird/examples/migrations/1185749658_add_city_to_users.down.sql @@ -0,0 +1 @@ +ALTER TABLE users DROP city; diff --git a/database/firebird/examples/migrations/1185749658_add_city_to_users.up.sql b/database/firebird/examples/migrations/1185749658_add_city_to_users.up.sql new file mode 100644 index 000000000..2add820be --- /dev/null +++ b/database/firebird/examples/migrations/1185749658_add_city_to_users.up.sql @@ -0,0 +1,3 @@ +ALTER TABLE users ADD city varchar(100); + + diff --git a/database/firebird/examples/migrations/1285849751_add_index_on_user_emails.down.sql b/database/firebird/examples/migrations/1285849751_add_index_on_user_emails.down.sql new file mode 100644 index 000000000..867ffb450 --- /dev/null +++ b/database/firebird/examples/migrations/1285849751_add_index_on_user_emails.down.sql @@ -0,0 +1 @@ +DROP INDEX users_email_index; diff --git a/database/firebird/examples/migrations/1285849751_add_index_on_user_emails.up.sql b/database/firebird/examples/migrations/1285849751_add_index_on_user_emails.up.sql new file mode 100644 index 000000000..03a04639c --- /dev/null +++ b/database/firebird/examples/migrations/1285849751_add_index_on_user_emails.up.sql @@ -0,0 +1,3 @@ +CREATE UNIQUE INDEX users_email_index ON users (email); + +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/firebird/examples/migrations/1385949617_create_books_table.down.sql b/database/firebird/examples/migrations/1385949617_create_books_table.down.sql new file mode 100644 index 000000000..3bd92c6ae --- /dev/null +++ b/database/firebird/examples/migrations/1385949617_create_books_table.down.sql @@ -0,0 +1 @@ +DROP TABLE books; diff --git a/database/firebird/examples/migrations/1385949617_create_books_table.up.sql b/database/firebird/examples/migrations/1385949617_create_books_table.up.sql new file mode 100644 index 000000000..f1503b518 --- /dev/null +++ b/database/firebird/examples/migrations/1385949617_create_books_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE books ( + user_id integer, + name varchar(40), + author varchar(40) +); diff --git a/database/firebird/examples/migrations/1485949617_create_movies_table.down.sql b/database/firebird/examples/migrations/1485949617_create_movies_table.down.sql new file mode 100644 index 000000000..6f5118f54 --- /dev/null +++ b/database/firebird/examples/migrations/1485949617_create_movies_table.down.sql @@ -0,0 +1 @@ +DROP TABLE movies; diff --git a/database/firebird/examples/migrations/1485949617_create_movies_table.up.sql b/database/firebird/examples/migrations/1485949617_create_movies_table.up.sql new file mode 100644 index 000000000..f0ef5943b --- /dev/null +++ b/database/firebird/examples/migrations/1485949617_create_movies_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE movies ( + user_id integer, + name varchar(40), + director varchar(40) +); diff --git a/database/firebird/firebird.go b/database/firebird/firebird.go new file mode 100644 index 000000000..9ee833649 --- /dev/null +++ b/database/firebird/firebird.go @@ -0,0 +1,259 @@ +//go:build go1.9 +// +build go1.9 + +package firebird + +import ( + "context" + "database/sql" + "fmt" + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database" + "github.com/hashicorp/go-multierror" + _ "github.com/nakagami/firebirdsql" + "go.uber.org/atomic" + "io" + "io/ioutil" + nurl "net/url" +) + +func init() { + db := Firebird{} + database.Register("firebird", &db) + database.Register("firebirdsql", &db) +} + +var DefaultMigrationsTable = "schema_migrations" + +var ( + ErrNilConfig = fmt.Errorf("no config") +) + +type Config struct { + DatabaseName string + MigrationsTable string +} + +type Firebird struct { + // Locking and unlocking need to use the same connection + conn *sql.Conn + db *sql.DB + isLocked atomic.Bool + + // Open and WithInstance need to guarantee that config is never nil + config *Config +} + +func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { + if config == nil { + return nil, ErrNilConfig + } + + if err := instance.Ping(); err != nil { + return nil, err + } + + if len(config.MigrationsTable) == 0 { + config.MigrationsTable = DefaultMigrationsTable + } + + conn, err := instance.Conn(context.Background()) + if err != nil { + return nil, err + } + + fb := &Firebird{ + conn: conn, + db: instance, + config: config, + } + + if err := fb.ensureVersionTable(); err != nil { + return nil, err + } + + return fb, nil +} + +func (f *Firebird) Open(dsn string) (database.Driver, error) { + purl, err := nurl.Parse(dsn) + if err != nil { + return nil, err + } + + db, err := sql.Open("firebirdsql", migrate.FilterCustomQuery(purl).String()) + if err != nil { + return nil, err + } + + px, err := WithInstance(db, &Config{ + MigrationsTable: purl.Query().Get("x-migrations-table"), + DatabaseName: purl.Path, + }) + + if err != nil { + return nil, err + } + + return px, nil +} + +func (f *Firebird) Close() error { + connErr := f.conn.Close() + dbErr := f.db.Close() + if connErr != nil || dbErr != nil { + return fmt.Errorf("conn: %v, db: %v", connErr, dbErr) + } + return nil +} + +func (f *Firebird) Lock() error { + if !f.isLocked.CAS(false, true) { + return database.ErrLocked + } + return nil +} + +func (f *Firebird) Unlock() error { + if !f.isLocked.CAS(true, false) { + return database.ErrNotLocked + } + return nil +} + +func (f *Firebird) Run(migration io.Reader) error { + migr, err := ioutil.ReadAll(migration) + if err != nil { + return err + } + + // run migration + query := string(migr[:]) + if _, err := f.conn.ExecContext(context.Background(), query); err != nil { + return database.Error{OrigErr: err, Err: "migration failed", Query: migr} + } + + return nil +} + +func (f *Firebird) SetVersion(version int, dirty bool) error { + // Always re-write the schema version to prevent empty schema version + // for failed down migration on the first migration + // See: https://github.com/golang-migrate/migrate/issues/330 + + // TODO: parameterize this SQL statement + // https://firebirdsql.org/refdocs/langrefupd20-execblock.html + // VALUES (?, ?) doesn't work + query := fmt.Sprintf(`EXECUTE BLOCK AS BEGIN + DELETE FROM "%v"; + INSERT INTO "%v" (version, dirty) VALUES (%v, %v); + END;`, + f.config.MigrationsTable, f.config.MigrationsTable, version, btoi(dirty)) + + if _, err := f.conn.ExecContext(context.Background(), query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + return nil +} + +func (f *Firebird) Version() (version int, dirty bool, err error) { + var d int + query := fmt.Sprintf(`SELECT FIRST 1 version, dirty FROM "%v"`, f.config.MigrationsTable) + err = f.conn.QueryRowContext(context.Background(), query).Scan(&version, &d) + switch { + case err == sql.ErrNoRows: + return database.NilVersion, false, nil + case err != nil: + return 0, false, &database.Error{OrigErr: err, Query: []byte(query)} + + default: + return version, itob(d), nil + } +} + +func (f *Firebird) Drop() (err error) { + // select all tables + query := `SELECT rdb$relation_name FROM rdb$relations WHERE rdb$view_blr IS NULL AND (rdb$system_flag IS NULL OR rdb$system_flag = 0);` + tables, err := f.conn.QueryContext(context.Background(), query) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + defer func() { + if errClose := tables.Close(); errClose != nil { + err = multierror.Append(err, errClose) + } + }() + + // delete one table after another + tableNames := make([]string, 0) + for tables.Next() { + var tableName string + if err := tables.Scan(&tableName); err != nil { + return err + } + if len(tableName) > 0 { + tableNames = append(tableNames, tableName) + } + } + if err := tables.Err(); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + // delete one by one ... + for _, t := range tableNames { + query := fmt.Sprintf(`EXECUTE BLOCK AS BEGIN + if (not exists(select 1 from rdb$relations where rdb$relation_name = '%v')) then + execute statement 'drop table "%v"'; + END;`, + t, t) + + if _, err := f.conn.ExecContext(context.Background(), query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + + return nil +} + +// ensureVersionTable checks if versions table exists and, if not, creates it. +func (f *Firebird) ensureVersionTable() (err error) { + if err = f.Lock(); err != nil { + return err + } + + defer func() { + if e := f.Unlock(); e != nil { + if err == nil { + err = e + } else { + err = multierror.Append(err, e) + } + } + }() + + query := fmt.Sprintf(`EXECUTE BLOCK AS BEGIN + if (not exists(select 1 from rdb$relations where rdb$relation_name = '%v')) then + execute statement 'create table "%v" (version bigint not null primary key, dirty smallint not null)'; + END;`, + f.config.MigrationsTable, f.config.MigrationsTable) + + if _, err = f.conn.ExecContext(context.Background(), query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + return nil +} + +// btoi converts bool to int +func btoi(v bool) int { + if v { + return 1 + } + return 0 +} + +// itob converts int to bool +func itob(v int) bool { + return v != 0 +} diff --git a/database/firebird/firebird_test.go b/database/firebird/firebird_test.go new file mode 100644 index 000000000..21fa8de2d --- /dev/null +++ b/database/firebird/firebird_test.go @@ -0,0 +1,226 @@ +package firebird + +import ( + "context" + "database/sql" + sqldriver "database/sql/driver" + "fmt" + "log" + + "github.com/golang-migrate/migrate/v4" + "io" + "strings" + "testing" + + "github.com/dhui/dktest" + + dt "github.com/golang-migrate/migrate/v4/database/testing" + "github.com/golang-migrate/migrate/v4/dktesting" + _ "github.com/golang-migrate/migrate/v4/source/file" + + _ "github.com/nakagami/firebirdsql" +) + +const ( + user = "test_user" + password = "123456" + dbName = "test.fdb" +) + +var ( + opts = dktest.Options{ + PortRequired: true, + ReadyFunc: isReady, + Env: map[string]string{ + "FIREBIRD_DATABASE": dbName, + "FIREBIRD_USER": user, + "FIREBIRD_PASSWORD": password, + }, + } + specs = []dktesting.ContainerSpec{ + {ImageName: "jacobalberty/firebird:2.5-ss", Options: opts}, + {ImageName: "jacobalberty/firebird:3.0", Options: opts}, + } +) + +func fbConnectionString(host, port string) string { + //firebird://user:password@servername[:port_number]/database_name_or_file[?params1=value1[¶m2=value2]...] + return fmt.Sprintf("firebird://%s:%s@%s:%s//firebird/data/%s", user, password, host, port, dbName) +} + +func isReady(ctx context.Context, c dktest.ContainerInfo) bool { + ip, port, err := c.FirstPort() + if err != nil { + return false + } + + db, err := sql.Open("firebirdsql", fbConnectionString(ip, port)) + if err != nil { + log.Println("open error:", err) + return false + } + defer func() { + if err := db.Close(); err != nil { + log.Println("close error:", err) + } + }() + if err = db.PingContext(ctx); err != nil { + switch err { + case sqldriver.ErrBadConn, io.EOF: + return false + default: + log.Println(err) + } + return false + } + + return true +} + +func Test(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := fbConnectionString(ip, port) + p := &Firebird{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + dt.Test(t, d, []byte("SELECT Count(*) FROM rdb$relations")) + }) +} + +func TestMigrate(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := fbConnectionString(ip, port) + p := &Firebird{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "firebirdsql", d) + if err != nil { + t.Fatal(err) + } + dt.TestMigrate(t, m) + }) +} + +func TestErrorParsing(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := fbConnectionString(ip, port) + p := &Firebird{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + + wantErr := `migration failed in line 0: CREATE TABLEE foo (foo varchar(40)); (details: Dynamic SQL Error +SQL error code = -104 +Token unknown - line 1, column 8 +TABLEE +)` + + if err := d.Run(strings.NewReader("CREATE TABLEE foo (foo varchar(40));")); err == nil { + t.Fatal("expected err but got nil") + } else if err.Error() != wantErr { + msg := err.Error() + t.Fatalf("expected '%s' but got '%s'", wantErr, msg) + } + }) +} + +func TestFilterCustomQuery(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := fbConnectionString(ip, port) + "?sslmode=disable&x-custom=foobar" + p := &Firebird{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + }) +} + +func Test_Lock(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := fbConnectionString(ip, port) + p := &Firebird{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + + dt.Test(t, d, []byte("SELECT Count(*) FROM rdb$relations")) + + ps := d.(*Firebird) + + err = ps.Lock() + if err != nil { + t.Fatal(err) + } + + err = ps.Unlock() + if err != nil { + t.Fatal(err) + } + + err = ps.Lock() + if err != nil { + t.Fatal(err) + } + + err = ps.Unlock() + if err != nil { + t.Fatal(err) + } + }) +} diff --git a/database/mongodb/README.md b/database/mongodb/README.md index e69de29bb..bbebe02cd 100644 --- a/database/mongodb/README.md +++ b/database/mongodb/README.md @@ -0,0 +1,24 @@ +# MongoDB + +* Driver work with mongo through [db.runCommands](https://docs.mongodb.com/manual/reference/command/) +* Migrations support json format. It contains array of commands for `db.runCommand`. Every command is executed in separate request to database +* All keys have to be in quotes `"` +* [Examples](./examples) + +# Usage + +`mongodb://user:password@host:port/dbname?query` (`mongodb+srv://` also works, but behaves a bit differently. See [docs](https://docs.mongodb.com/manual/reference/connection-string/#dns-seedlist-connection-format) for more information) + +| URL Query | WithInstance Config | Description | +|------------|---------------------|-------------| +| `x-migrations-collection` | `MigrationsCollection` | Name of the migrations collection | +| `x-transaction-mode` | `TransactionMode` | If set to `true` wrap commands in [transaction](https://docs.mongodb.com/manual/core/transactions). Available only for replica set. Driver is using [strconv.ParseBool](https://golang.org/pkg/strconv/#ParseBool) for parsing| +| `x-advisory-locking` | `true` | Feature flag for advisory locking, if set to false, disable advisory locking | +| `x-advisory-lock-collection` | `migrate_advisory_lock` | The name of the collection to use for advisory locking.| +| `x-advisory-lock-timeout` | `15` | The max time in seconds that migrate will wait to acquire a lock before failing. | +| `x-advisory-lock-timeout-interval` | `10` | The max time in seconds between attempts to acquire the advisory lock, the lock is attempted to be acquired using an exponential backoff algorithm. | +| `dbname` | `DatabaseName` | The name of the database to connect to | +| `user` | | The user to sign in as. Can be omitted | +| `password` | | The user's password. Can be omitted | +| `host` | | The host to connect to | +| `port` | | The port to bind to | \ No newline at end of file diff --git a/database/mongodb/examples/migrations/001_create_user.down.json b/database/mongodb/examples/migrations/001_create_user.down.json new file mode 100644 index 000000000..6bba284e5 --- /dev/null +++ b/database/mongodb/examples/migrations/001_create_user.down.json @@ -0,0 +1,5 @@ +[ + { + "dropUser": "deminem" + } +] \ No newline at end of file diff --git a/database/mongodb/examples/migrations/001_create_user.up.json b/database/mongodb/examples/migrations/001_create_user.up.json new file mode 100644 index 000000000..6c37cb702 --- /dev/null +++ b/database/mongodb/examples/migrations/001_create_user.up.json @@ -0,0 +1,12 @@ +[ + { + "createUser": "deminem", + "pwd": "gogo", + "roles": [ + { + "role": "readWrite", + "db": "testMigration" + } + ] + } +] \ No newline at end of file diff --git a/database/mongodb/examples/migrations/002_create_indexes.down.json b/database/mongodb/examples/migrations/002_create_indexes.down.json new file mode 100644 index 000000000..6bba481a6 --- /dev/null +++ b/database/mongodb/examples/migrations/002_create_indexes.down.json @@ -0,0 +1,10 @@ +[ + { + "dropIndexes": "mycollection", + "index": "username_sort_by_asc_created" + }, + { + "dropIndexes": "mycollection", + "index": "unique_email" + } +] \ No newline at end of file diff --git a/database/mongodb/examples/migrations/002_create_indexes.up.json b/database/mongodb/examples/migrations/002_create_indexes.up.json new file mode 100644 index 000000000..e2995a20f --- /dev/null +++ b/database/mongodb/examples/migrations/002_create_indexes.up.json @@ -0,0 +1,21 @@ +[{ + "createIndexes": "mycollection", + "indexes": [ + { + "key": { + "username": 1, + "created": -1 + }, + "name": "username_sort_by_asc_created", + "background": true + }, + { + "key": { + "email": 1 + }, + "name": "unique_email", + "unique": true, + "background": true + } + ] +}] \ No newline at end of file diff --git a/database/mongodb/examples/migrations/003_add_new_field.down.json b/database/mongodb/examples/migrations/003_add_new_field.down.json new file mode 100644 index 000000000..506c863cd --- /dev/null +++ b/database/mongodb/examples/migrations/003_add_new_field.down.json @@ -0,0 +1,16 @@ +[ + { + "update": "users", + "updates": [ + { + "q": {}, + "u": { + "$unset": { + "status": "" + } + }, + "multi": true + } + ] + } +] diff --git a/database/mongodb/examples/migrations/003_add_new_field.up.json b/database/mongodb/examples/migrations/003_add_new_field.up.json new file mode 100644 index 000000000..6f53995e3 --- /dev/null +++ b/database/mongodb/examples/migrations/003_add_new_field.up.json @@ -0,0 +1,16 @@ +[ + { + "update": "users", + "updates": [ + { + "q": {}, + "u": { + "$set": { + "status": "active" + } + }, + "multi": true + } + ] + } +] diff --git a/database/mongodb/examples/migrations/004_replace_field_value_from_another_field.down.json b/database/mongodb/examples/migrations/004_replace_field_value_from_another_field.down.json new file mode 100644 index 000000000..0d2e65ebe --- /dev/null +++ b/database/mongodb/examples/migrations/004_replace_field_value_from_another_field.down.json @@ -0,0 +1,14 @@ +[ + { + "update": "users", + "updates": [ + { + "q": {}, + "u": { + "fullname": "" + }, + "multi": true + } + ] + } +] diff --git a/database/mongodb/examples/migrations/004_replace_field_value_from_another_field.up.json b/database/mongodb/examples/migrations/004_replace_field_value_from_another_field.up.json new file mode 100644 index 000000000..1805e2688 --- /dev/null +++ b/database/mongodb/examples/migrations/004_replace_field_value_from_another_field.up.json @@ -0,0 +1,23 @@ +[ + { + "aggregate": "users", + "pipeline": [ + { + "$project": { + "_id": 1, + "firstname": 1, + "lastname": 1, + "username": 1, + "password": 1, + "email": 1, + "active": 1, + "fullname": { "$concat": ["$firstname", " ", "$lastname"] } + } + }, + { + "$out": "users" + } + ], + "cursor": {} + } +] diff --git a/database/mongodb/mongodb.go b/database/mongodb/mongodb.go new file mode 100644 index 000000000..00f0fd1be --- /dev/null +++ b/database/mongodb/mongodb.go @@ -0,0 +1,404 @@ +package mongodb + +import ( + "context" + "fmt" + "github.com/cenkalti/backoff/v4" + "github.com/golang-migrate/migrate/v4/database" + "github.com/hashicorp/go-multierror" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/x/mongo/driver/connstring" + "go.uber.org/atomic" + "io" + "io/ioutil" + "net/url" + os "os" + "strconv" + "time" +) + +func init() { + db := Mongo{} + database.Register("mongodb", &db) + database.Register("mongodb+srv", &db) +} + +var DefaultMigrationsCollection = "schema_migrations" + +const DefaultLockingCollection = "migrate_advisory_lock" // the collection to use for advisory locking by default. +const lockKeyUniqueValue = 0 // the unique value to lock on. If multiple clients try to insert the same key, it will fail (locked). +const DefaultLockTimeout = 15 // the default maximum time to wait for a lock to be released. +const DefaultLockTimeoutInterval = 10 // the default maximum intervals time for the locking timout. +const DefaultAdvisoryLockingFlag = true // the default value for the advisory locking feature flag. Default is true. +const LockIndexName = "lock_unique_key" // the name of the index which adds unique constraint to the locking_key field. +const contextWaitTimeout = 5 * time.Second // how long to wait for the request to mongo to block/wait for. + +var ( + ErrNoDatabaseName = fmt.Errorf("no database name") + ErrNilConfig = fmt.Errorf("no config") + ErrLockTimeoutConfigConflict = fmt.Errorf("both x-advisory-lock-timeout-interval and x-advisory-lock-timout-interval were specified") +) + +type Mongo struct { + client *mongo.Client + db *mongo.Database + config *Config + isLocked atomic.Bool +} + +type Locking struct { + CollectionName string + Timeout int + Enabled bool + Interval int +} +type Config struct { + DatabaseName string + MigrationsCollection string + TransactionMode bool + Locking Locking +} +type versionInfo struct { + Version int `bson:"version"` + Dirty bool `bson:"dirty"` +} + +type lockObj struct { + Key int `bson:"locking_key"` + Pid int `bson:"pid"` + Hostname string `bson:"hostname"` + CreatedAt time.Time `bson:"created_at"` +} +type findFilter struct { + Key int `bson:"locking_key"` +} + +func WithInstance(instance *mongo.Client, config *Config) (database.Driver, error) { + if config == nil { + return nil, ErrNilConfig + } + if len(config.DatabaseName) == 0 { + return nil, ErrNoDatabaseName + } + if len(config.MigrationsCollection) == 0 { + config.MigrationsCollection = DefaultMigrationsCollection + } + if len(config.Locking.CollectionName) == 0 { + config.Locking.CollectionName = DefaultLockingCollection + } + if config.Locking.Timeout <= 0 { + config.Locking.Timeout = DefaultLockTimeout + } + if config.Locking.Interval <= 0 { + config.Locking.Interval = DefaultLockTimeoutInterval + } + + mc := &Mongo{ + client: instance, + db: instance.Database(config.DatabaseName), + config: config, + } + + if mc.config.Locking.Enabled { + if err := mc.ensureLockTable(); err != nil { + return nil, err + } + } + if err := mc.ensureVersionTable(); err != nil { + return nil, err + } + + return mc, nil +} + +func (m *Mongo) Open(dsn string) (database.Driver, error) { + //connstring is experimental package, but it used for parse connection string in mongo.Connect function + uri, err := connstring.Parse(dsn) + if err != nil { + return nil, err + } + if len(uri.Database) == 0 { + return nil, ErrNoDatabaseName + } + unknown := url.Values(uri.UnknownOptions) + + migrationsCollection := unknown.Get("x-migrations-collection") + lockCollection := unknown.Get("x-advisory-lock-collection") + transactionMode, err := parseBoolean(unknown.Get("x-transaction-mode"), false) + if err != nil { + return nil, err + } + advisoryLockingFlag, err := parseBoolean(unknown.Get("x-advisory-locking"), DefaultAdvisoryLockingFlag) + if err != nil { + return nil, err + } + lockingTimout, err := parseInt(unknown.Get("x-advisory-lock-timeout"), DefaultLockTimeout) + if err != nil { + return nil, err + } + + lockTimeoutIntervalValue := unknown.Get("x-advisory-lock-timeout-interval") + // The initial release had a typo for this argument but for backwards compatibility sake, we will keep supporting it + // and we will error out if both values are set. + lockTimeoutIntervalValueFromTypo := unknown.Get("x-advisory-lock-timout-interval") + + lockTimeout := lockTimeoutIntervalValue + + if lockTimeoutIntervalValue != "" && lockTimeoutIntervalValueFromTypo != "" { + return nil, ErrLockTimeoutConfigConflict + } else if lockTimeoutIntervalValueFromTypo != "" { + lockTimeout = lockTimeoutIntervalValueFromTypo + } + + maxLockCheckInterval, err := parseInt(lockTimeout, DefaultLockTimeoutInterval) + + if err != nil { + return nil, err + } + client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(dsn)) + if err != nil { + return nil, err + } + + if err = client.Ping(context.TODO(), nil); err != nil { + return nil, err + } + mc, err := WithInstance(client, &Config{ + DatabaseName: uri.Database, + MigrationsCollection: migrationsCollection, + TransactionMode: transactionMode, + Locking: Locking{ + CollectionName: lockCollection, + Timeout: lockingTimout, + Enabled: advisoryLockingFlag, + Interval: maxLockCheckInterval, + }, + }) + if err != nil { + return nil, err + } + return mc, nil +} + +//Parse the url param, convert it to boolean +// returns error if param invalid. returns defaultValue if param not present +func parseBoolean(urlParam string, defaultValue bool) (bool, error) { + + // if parameter passed, parse it (otherwise return default value) + if urlParam != "" { + result, err := strconv.ParseBool(urlParam) + if err != nil { + return false, err + } + return result, nil + } + + // if no url Param passed, return default value + return defaultValue, nil +} + +//Parse the url param, convert it to int +// returns error if param invalid. returns defaultValue if param not present +func parseInt(urlParam string, defaultValue int) (int, error) { + + // if parameter passed, parse it (otherwise return default value) + if urlParam != "" { + result, err := strconv.Atoi(urlParam) + if err != nil { + return -1, err + } + return result, nil + } + + // if no url Param passed, return default value + return defaultValue, nil +} +func (m *Mongo) SetVersion(version int, dirty bool) error { + migrationsCollection := m.db.Collection(m.config.MigrationsCollection) + if err := migrationsCollection.Drop(context.TODO()); err != nil { + return &database.Error{OrigErr: err, Err: "drop migrations collection failed"} + } + _, err := migrationsCollection.InsertOne(context.TODO(), bson.M{"version": version, "dirty": dirty}) + if err != nil { + return &database.Error{OrigErr: err, Err: "save version failed"} + } + return nil +} + +func (m *Mongo) Version() (version int, dirty bool, err error) { + var versionInfo versionInfo + err = m.db.Collection(m.config.MigrationsCollection).FindOne(context.TODO(), bson.M{}).Decode(&versionInfo) + switch { + case err == mongo.ErrNoDocuments: + return database.NilVersion, false, nil + case err != nil: + return 0, false, &database.Error{OrigErr: err, Err: "failed to get migration version"} + default: + return versionInfo.Version, versionInfo.Dirty, nil + } +} + +func (m *Mongo) Run(migration io.Reader) error { + migr, err := ioutil.ReadAll(migration) + if err != nil { + return err + } + var cmds []bson.D + err = bson.UnmarshalExtJSON(migr, true, &cmds) + if err != nil { + return fmt.Errorf("unmarshaling json error: %s", err) + } + if m.config.TransactionMode { + if err := m.executeCommandsWithTransaction(context.TODO(), cmds); err != nil { + return err + } + } else { + if err := m.executeCommands(context.TODO(), cmds); err != nil { + return err + } + } + return nil +} + +func (m *Mongo) executeCommandsWithTransaction(ctx context.Context, cmds []bson.D) error { + err := m.db.Client().UseSession(ctx, func(sessionContext mongo.SessionContext) error { + if err := sessionContext.StartTransaction(); err != nil { + return &database.Error{OrigErr: err, Err: "failed to start transaction"} + } + if err := m.executeCommands(sessionContext, cmds); err != nil { + //When command execution is failed, it's aborting transaction + //If you tried to call abortTransaction, it`s return error that transaction already aborted + return err + } + if err := sessionContext.CommitTransaction(sessionContext); err != nil { + return &database.Error{OrigErr: err, Err: "failed to commit transaction"} + } + return nil + }) + if err != nil { + return err + } + return nil +} + +func (m *Mongo) executeCommands(ctx context.Context, cmds []bson.D) error { + for _, cmd := range cmds { + err := m.db.RunCommand(ctx, cmd).Err() + if err != nil { + return &database.Error{OrigErr: err, Err: fmt.Sprintf("failed to execute command:%v", cmd)} + } + } + return nil +} + +func (m *Mongo) Close() error { + return m.client.Disconnect(context.TODO()) +} + +func (m *Mongo) Drop() error { + return m.db.Drop(context.TODO()) +} + +func (m *Mongo) ensureLockTable() error { + indexes := m.db.Collection(m.config.Locking.CollectionName).Indexes() + + indexOptions := options.Index().SetUnique(true).SetName(LockIndexName) + _, err := indexes.CreateOne(context.TODO(), mongo.IndexModel{ + Options: indexOptions, + Keys: findFilter{Key: -1}, + }) + if err != nil { + return err + } + return nil +} + +// ensureVersionTable checks if versions table exists and, if not, creates it. +// Note that this function locks the database, which deviates from the usual +// convention of "caller locks" in the MongoDb type. +func (m *Mongo) ensureVersionTable() (err error) { + if err = m.Lock(); err != nil { + return err + } + + defer func() { + if e := m.Unlock(); e != nil { + if err == nil { + err = e + } else { + err = multierror.Append(err, e) + } + } + }() + + if err != nil { + return err + } + if _, _, err = m.Version(); err != nil { + return err + } + return nil +} + +// Utilizes advisory locking on the config.LockingCollection collection +// This uses a unique index on the `locking_key` field. +func (m *Mongo) Lock() error { + return database.CasRestoreOnErr(&m.isLocked, false, true, database.ErrLocked, func() error { + if !m.config.Locking.Enabled { + return nil + } + + pid := os.Getpid() + hostname, err := os.Hostname() + if err != nil { + hostname = fmt.Sprintf("Could not determine hostname. Error: %s", err.Error()) + } + + newLockObj := lockObj{ + Key: lockKeyUniqueValue, + Pid: pid, + Hostname: hostname, + CreatedAt: time.Now(), + } + operation := func() error { + timeout, cancelFunc := context.WithTimeout(context.Background(), contextWaitTimeout) + _, err := m.db.Collection(m.config.Locking.CollectionName).InsertOne(timeout, newLockObj) + defer cancelFunc() + return err + } + exponentialBackOff := backoff.NewExponentialBackOff() + duration := time.Duration(m.config.Locking.Timeout) * time.Second + exponentialBackOff.MaxElapsedTime = duration + exponentialBackOff.MaxInterval = time.Duration(m.config.Locking.Interval) * time.Second + + err = backoff.Retry(operation, exponentialBackOff) + if err != nil { + return database.ErrLocked + } + + return nil + }) +} + +func (m *Mongo) Unlock() error { + return database.CasRestoreOnErr(&m.isLocked, true, false, database.ErrNotLocked, func() error { + if !m.config.Locking.Enabled { + return nil + } + + filter := findFilter{ + Key: lockKeyUniqueValue, + } + + ctx, cancel := context.WithTimeout(context.Background(), contextWaitTimeout) + _, err := m.db.Collection(m.config.Locking.CollectionName).DeleteMany(ctx, filter) + defer cancel() + + if err != nil { + return err + } + return nil + }) +} diff --git a/database/mongodb/mongodb_test.go b/database/mongodb/mongodb_test.go new file mode 100644 index 000000000..f15f74113 --- /dev/null +++ b/database/mongodb/mongodb_test.go @@ -0,0 +1,405 @@ +package mongodb + +import ( + "bytes" + "context" + "fmt" + + "log" + + "github.com/golang-migrate/migrate/v4" + "io" + "os" + "strconv" + "testing" + "time" +) + +import ( + "github.com/dhui/dktest" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +import ( + dt "github.com/golang-migrate/migrate/v4/database/testing" + "github.com/golang-migrate/migrate/v4/dktesting" + _ "github.com/golang-migrate/migrate/v4/source/file" +) + +var ( + opts = dktest.Options{PortRequired: true, ReadyFunc: isReady} + // Supported versions: https://www.mongodb.com/support-policy + specs = []dktesting.ContainerSpec{ + {ImageName: "mongo:3.4", Options: opts}, + {ImageName: "mongo:3.6", Options: opts}, + {ImageName: "mongo:4.0", Options: opts}, + {ImageName: "mongo:4.2", Options: opts}, + } +) + +func mongoConnectionString(host, port string) string { + // there is connect option for excluding serverConnection algorithm + // it's let avoid errors with mongo replica set connection in docker container + return fmt.Sprintf("mongodb://%s:%s/testMigration?connect=direct", host, port) +} + +func isReady(ctx context.Context, c dktest.ContainerInfo) bool { + ip, port, err := c.FirstPort() + if err != nil { + return false + } + + client, err := mongo.Connect(ctx, options.Client().ApplyURI(mongoConnectionString(ip, port))) + if err != nil { + return false + } + defer func() { + if err := client.Disconnect(ctx); err != nil { + log.Println("close error:", err) + } + }() + + if err = client.Ping(ctx, nil); err != nil { + switch err { + case io.EOF: + return false + default: + log.Println(err) + } + return false + } + return true +} + +func Test(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := mongoConnectionString(ip, port) + p := &Mongo{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + dt.TestNilVersion(t, d) + dt.TestLockAndUnlock(t, d) + dt.TestRun(t, d, bytes.NewReader([]byte(`[{"insert":"hello","documents":[{"wild":"world"}]}]`))) + dt.TestSetVersion(t, d) + dt.TestDrop(t, d) + }) +} + +func TestMigrate(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := mongoConnectionString(ip, port) + p := &Mongo{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "", d) + if err != nil { + t.Fatal(err) + } + dt.TestMigrate(t, m) + }) +} + +func TestWithAuth(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := mongoConnectionString(ip, port) + p := &Mongo{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + createUserCMD := []byte(`[{"createUser":"deminem","pwd":"gogo","roles":[{"role":"readWrite","db":"testMigration"}]}]`) + err = d.Run(bytes.NewReader(createUserCMD)) + if err != nil { + t.Fatal(err) + } + testcases := []struct { + name string + connectUri string + isErrorExpected bool + }{ + {"right auth data", "mongodb://deminem:gogo@%s:%v/testMigration", false}, + {"wrong auth data", "mongodb://wrong:auth@%s:%v/testMigration", true}, + } + + for _, tcase := range testcases { + t.Run(tcase.name, func(t *testing.T) { + mc := &Mongo{} + d, err := mc.Open(fmt.Sprintf(tcase.connectUri, ip, port)) + if err == nil { + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + } + + switch { + case tcase.isErrorExpected && err == nil: + t.Fatalf("no error when expected") + case !tcase.isErrorExpected && err != nil: + t.Fatalf("unexpected error: %v", err) + } + }) + } + }) +} + +func TestLockWorks(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := mongoConnectionString(ip, port) + p := &Mongo{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + + dt.TestRun(t, d, bytes.NewReader([]byte(`[{"insert":"hello","documents":[{"wild":"world"}]}]`))) + + mc := d.(*Mongo) + + err = mc.Lock() + if err != nil { + t.Fatal(err) + } + err = mc.Unlock() + if err != nil { + t.Fatal(err) + } + + err = mc.Lock() + if err != nil { + t.Fatal(err) + } + err = mc.Unlock() + if err != nil { + t.Fatal(err) + } + + // enable locking, + //try to hit a lock conflict + mc.config.Locking.Enabled = true + mc.config.Locking.Timeout = 1 + err = mc.Lock() + if err != nil { + t.Fatal(err) + } + err = mc.Lock() + if err == nil { + t.Fatal("should have failed, mongo should be locked already") + } + }) +} + +func TestTransaction(t *testing.T) { + transactionSpecs := []dktesting.ContainerSpec{ + {ImageName: "mongo:4", Options: dktest.Options{PortRequired: true, ReadyFunc: isReady, + Cmd: []string{"mongod", "--bind_ip_all", "--replSet", "rs0"}}}, + } + dktesting.ParallelTest(t, transactionSpecs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(mongoConnectionString(ip, port))) + if err != nil { + t.Fatal(err) + } + err = client.Ping(context.TODO(), nil) + if err != nil { + t.Fatal(err) + } + //rs.initiate() + err = client.Database("admin").RunCommand(context.TODO(), bson.D{bson.E{Key: "replSetInitiate", Value: bson.D{}}}).Err() + if err != nil { + t.Fatal(err) + } + err = waitForReplicaInit(client) + if err != nil { + t.Fatal(err) + } + d, err := WithInstance(client, &Config{ + DatabaseName: "testMigration", + }) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + //We have to create collection + //transactions don't support operations with creating new dbs, collections + //Unique index need for checking transaction aborting + insertCMD := []byte(`[ + {"create":"hello"}, + {"createIndexes": "hello", + "indexes": [{ + "key": { + "wild": 1 + }, + "name": "unique_wild", + "unique": true, + "background": true + }] + }]`) + err = d.Run(bytes.NewReader(insertCMD)) + if err != nil { + t.Fatal(err) + } + testcases := []struct { + name string + cmds []byte + documentsCount int64 + isErrorExpected bool + }{ + { + name: "success transaction", + cmds: []byte(`[{"insert":"hello","documents":[ + {"wild":"world"}, + {"wild":"west"}, + {"wild":"natural"} + ] + }]`), + documentsCount: 3, + isErrorExpected: false, + }, + { + name: "failure transaction", + //transaction have to be failure - duplicate unique key wild:west + //none of the documents should be added + cmds: []byte(`[{"insert":"hello","documents":[{"wild":"flower"}]}, + {"insert":"hello","documents":[ + {"wild":"cat"}, + {"wild":"west"} + ] + }]`), + documentsCount: 3, + isErrorExpected: true, + }, + } + for _, tcase := range testcases { + t.Run(tcase.name, func(t *testing.T) { + client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(mongoConnectionString(ip, port))) + if err != nil { + t.Fatal(err) + } + err = client.Ping(context.TODO(), nil) + if err != nil { + t.Fatal(err) + } + d, err := WithInstance(client, &Config{ + DatabaseName: "testMigration", + TransactionMode: true, + }) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + runErr := d.Run(bytes.NewReader(tcase.cmds)) + if runErr != nil { + if !tcase.isErrorExpected { + t.Fatal(runErr) + } + } + documentsCount, err := client.Database("testMigration").Collection("hello").CountDocuments(context.TODO(), bson.M{}) + if err != nil { + t.Fatal(err) + } + if tcase.documentsCount != documentsCount { + t.Fatalf("expected %d and actual %d documents count not equal. run migration error:%s", tcase.documentsCount, documentsCount, runErr) + } + }) + } + }) +} + +type isMaster struct { + IsMaster bool `bson:"ismaster"` +} + +func waitForReplicaInit(client *mongo.Client) error { + ticker := time.NewTicker(time.Second * 1) + defer ticker.Stop() + timeout, err := strconv.Atoi(os.Getenv("MIGRATE_TEST_MONGO_REPLICA_SET_INIT_TIMEOUT")) + if err != nil { + timeout = 30 + } + timeoutTimer := time.NewTimer(time.Duration(timeout) * time.Second) + defer timeoutTimer.Stop() + for { + select { + case <-ticker.C: + var status isMaster + //Check that node is primary because + //during replica set initialization, the first node first becomes a secondary and then becomes the primary + //should consider that initialization is completed only after the node has become the primary + result := client.Database("admin").RunCommand(context.TODO(), bson.D{bson.E{Key: "isMaster", Value: 1}}) + r, err := result.DecodeBytes() + if err != nil { + return err + } + err = bson.Unmarshal(r, &status) + if err != nil { + return err + } + if status.IsMaster { + return nil + } + case <-timeoutTimer.C: + return fmt.Errorf("replica init timeout") + } + } + +} diff --git a/database/multistmt/parse.go b/database/multistmt/parse.go new file mode 100644 index 000000000..9a045767d --- /dev/null +++ b/database/multistmt/parse.go @@ -0,0 +1,46 @@ +// Package multistmt provides methods for parsing multi-statement database migrations +package multistmt + +import ( + "bufio" + "bytes" + "io" +) + +// StartBufSize is the default starting size of the buffer used to scan and parse multi-statement migrations +var StartBufSize = 4096 + +// Handler handles a single migration parsed from a multi-statement migration. +// It's given the single migration to handle and returns whether or not further statements +// from the multi-statement migration should be parsed and handled. +type Handler func(migration []byte) bool + +func splitWithDelimiter(delimiter []byte) func(d []byte, atEOF bool) (int, []byte, error) { + return func(d []byte, atEOF bool) (int, []byte, error) { + // SplitFunc inspired by bufio.ScanLines() implementation + if atEOF { + if len(d) == 0 { + return 0, nil, nil + } + return len(d), d, nil + } + if i := bytes.Index(d, delimiter); i >= 0 { + return i + len(delimiter), d[:i+len(delimiter)], nil + } + return 0, nil, nil + } +} + +// Parse parses the given multi-statement migration +func Parse(reader io.Reader, delimiter []byte, maxMigrationSize int, h Handler) error { + scanner := bufio.NewScanner(reader) + scanner.Buffer(make([]byte, 0, StartBufSize), maxMigrationSize) + scanner.Split(splitWithDelimiter(delimiter)) + for scanner.Scan() { + cont := h(scanner.Bytes()) + if !cont { + break + } + } + return scanner.Err() +} diff --git a/database/multistmt/parse_test.go b/database/multistmt/parse_test.go new file mode 100644 index 000000000..5d3e975c4 --- /dev/null +++ b/database/multistmt/parse_test.go @@ -0,0 +1,57 @@ +package multistmt_test + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/golang-migrate/migrate/v4/database/multistmt" +) + +const maxMigrationSize = 1024 + +func TestParse(t *testing.T) { + testCases := []struct { + name string + multiStmt string + delimiter string + expected []string + expectedErr error + }{ + {name: "single statement, no delimiter", multiStmt: "single statement, no delimiter", delimiter: ";", + expected: []string{"single statement, no delimiter"}, expectedErr: nil}, + {name: "single statement, one delimiter", multiStmt: "single statement, one delimiter;", delimiter: ";", + expected: []string{"single statement, one delimiter;"}, expectedErr: nil}, + {name: "two statements, no trailing delimiter", multiStmt: "statement one; statement two", delimiter: ";", + expected: []string{"statement one;", " statement two"}, expectedErr: nil}, + {name: "two statements, with trailing delimiter", multiStmt: "statement one; statement two;", delimiter: ";", + expected: []string{"statement one;", " statement two;"}, expectedErr: nil}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + stmts := make([]string, 0, len(tc.expected)) + err := multistmt.Parse(strings.NewReader(tc.multiStmt), []byte(tc.delimiter), maxMigrationSize, func(b []byte) bool { + stmts = append(stmts, string(b)) + return true + }) + assert.Equal(t, tc.expectedErr, err) + assert.Equal(t, tc.expected, stmts) + }) + } +} + +func TestParseDiscontinue(t *testing.T) { + multiStmt := "statement one; statement two" + delimiter := ";" + expected := []string{"statement one;"} + + stmts := make([]string, 0, len(expected)) + err := multistmt.Parse(strings.NewReader(multiStmt), []byte(delimiter), maxMigrationSize, func(b []byte) bool { + stmts = append(stmts, string(b)) + return false + }) + assert.Nil(t, err) + assert.Equal(t, expected, stmts) +} diff --git a/database/mysql/README.md b/database/mysql/README.md index d0b908d2d..096fa5e6b 100644 --- a/database/mysql/README.md +++ b/database/mysql/README.md @@ -5,6 +5,7 @@ | URL Query | WithInstance Config | Description | |------------|---------------------|-------------| | `x-migrations-table` | `MigrationsTable` | Name of the migrations table | +| `x-no-lock` | `NoLock` | Set to `true` to skip `GET_LOCK`/`RELEASE_LOCK` statements. Useful for [multi-master MySQL flavors](https://www.percona.com/doc/percona-xtradb-cluster/LATEST/features/pxc-strict-mode.html#explicit-table-locking). Only run migrations from one host when this is enabled. | | `dbname` | `DatabaseName` | The name of the database to connect to | | `user` | | The user to sign in as | | `password` | | The user's password | @@ -27,9 +28,9 @@ import ( "database/sql" _ "github.com/go-sql-driver/mysql" - "github.com/golang-migrate/migrate" - "github.com/golang-migrate/migrate/database/mysql" - _ "github.com/golang-migrate/migrate/source/file" + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database/mysql" + _ "github.com/golang-migrate/migrate/v4/source/file" ) func main() { diff --git a/database/mysql/examples/migrations/1_init.down.sql b/database/mysql/examples/migrations/1_init.down.sql new file mode 100644 index 000000000..1b10e6fc0 --- /dev/null +++ b/database/mysql/examples/migrations/1_init.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS test; \ No newline at end of file diff --git a/database/mysql/examples/migrations/1_init.up.sql b/database/mysql/examples/migrations/1_init.up.sql new file mode 100644 index 000000000..2c3d7a1f2 --- /dev/null +++ b/database/mysql/examples/migrations/1_init.up.sql @@ -0,0 +1,3 @@ +CREATE TABLE IF NOT EXISTS test ( + firstname VARCHAR(16) +); \ No newline at end of file diff --git a/database/mysql/mysql.go b/database/mysql/mysql.go index 20c840e02..14b15390e 100644 --- a/database/mysql/mysql.go +++ b/database/mysql/mysql.go @@ -1,3 +1,4 @@ +//go:build go1.9 // +build go1.9 package mysql @@ -8,22 +9,20 @@ import ( "crypto/x509" "database/sql" "fmt" + "go.uber.org/atomic" "io" "io/ioutil" nurl "net/url" "strconv" "strings" -) -import ( "github.com/go-sql-driver/mysql" -) - -import ( - "github.com/golang-migrate/migrate/v4" "github.com/golang-migrate/migrate/v4/database" + "github.com/hashicorp/go-multierror" ) +var _ database.Driver = (*Mysql)(nil) // explicit compile time type check + func init() { database.Register("mysql", &Mysql{}) } @@ -41,6 +40,7 @@ var ( type Config struct { MigrationsTable string DatabaseName string + NoLock bool } type Mysql struct { @@ -48,46 +48,43 @@ type Mysql struct { // just do everything over a single conn anyway. conn *sql.Conn db *sql.DB - isLocked bool + isLocked atomic.Bool config *Config } -// instance must have `multiStatements` set to true -func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { +// connection instance must have `multiStatements` set to true +func WithConnection(ctx context.Context, conn *sql.Conn, config *Config) (*Mysql, error) { if config == nil { return nil, ErrNilConfig } - if err := instance.Ping(); err != nil { + if err := conn.PingContext(ctx); err != nil { return nil, err } - query := `SELECT DATABASE()` - var databaseName sql.NullString - if err := instance.QueryRow(query).Scan(&databaseName); err != nil { - return nil, &database.Error{OrigErr: err, Query: []byte(query)} - } - - if len(databaseName.String) == 0 { - return nil, ErrNoDatabaseName + mx := &Mysql{ + conn: conn, + db: nil, + config: config, } - config.DatabaseName = databaseName.String + if config.DatabaseName == "" { + query := `SELECT DATABASE()` + var databaseName sql.NullString + if err := conn.QueryRowContext(ctx, query).Scan(&databaseName); err != nil { + return nil, &database.Error{OrigErr: err, Query: []byte(query)} + } - if len(config.MigrationsTable) == 0 { - config.MigrationsTable = DefaultMigrationsTable - } + if len(databaseName.String) == 0 { + return nil, ErrNoDatabaseName + } - conn, err := instance.Conn(context.Background()) - if err != nil { - return nil, err + config.DatabaseName = databaseName.String } - mx := &Mysql{ - conn: conn, - db: instance, - config: config, + if len(config.MigrationsTable) == 0 { + config.MigrationsTable = DefaultMigrationsTable } if err := mx.ensureVersionTable(); err != nil { @@ -97,95 +94,162 @@ func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { return mx, nil } -// urlToMySQLConfig takes a net/url URL and returns a go-sql-driver/mysql Config. -// Manually sets username and password to avoid net/url from url-encoding the reserved URL characters -func urlToMySQLConfig(u nurl.URL) (*mysql.Config, error) { - origUserInfo := u.User - u.User = nil +// instance must have `multiStatements` set to true +func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { + ctx := context.Background() - c, err := mysql.ParseDSN(strings.TrimPrefix(u.String(), "mysql://")) - if err != nil { + if err := instance.Ping(); err != nil { return nil, err } - if origUserInfo != nil { - c.User = origUserInfo.Username() - if p, ok := origUserInfo.Password(); ok { - c.Passwd = p - } + + conn, err := instance.Conn(ctx) + if err != nil { + return nil, err } - return c, nil -} -func (m *Mysql) Open(url string) (database.Driver, error) { - purl, err := nurl.Parse(url) + mx, err := WithConnection(ctx, conn, config) if err != nil { return nil, err } - q := purl.Query() - q.Set("multiStatements", "true") - purl.RawQuery = q.Encode() + mx.db = instance - migrationsTable := purl.Query().Get("x-migrations-table") - if len(migrationsTable) == 0 { - migrationsTable = DefaultMigrationsTable + return mx, nil +} + +// extractCustomQueryParams extracts the custom query params (ones that start with "x-") from +// mysql.Config.Params (connection parameters) as to not interfere with connecting to MySQL +func extractCustomQueryParams(c *mysql.Config) (map[string]string, error) { + if c == nil { + return nil, ErrNilConfig } + customQueryParams := map[string]string{} - // use custom TLS? - ctls := purl.Query().Get("tls") - if len(ctls) > 0 { - if _, isBool := readBool(ctls); !isBool && strings.ToLower(ctls) != "skip-verify" { - rootCertPool := x509.NewCertPool() - pem, err := ioutil.ReadFile(purl.Query().Get("x-tls-ca")) - if err != nil { - return nil, err - } + for k, v := range c.Params { + if strings.HasPrefix(k, "x-") { + customQueryParams[k] = v + delete(c.Params, k) + } + } + return customQueryParams, nil +} - if ok := rootCertPool.AppendCertsFromPEM(pem); !ok { - return nil, ErrAppendPEM - } +func urlToMySQLConfig(url string) (*mysql.Config, error) { + // Need to parse out custom TLS parameters and call + // mysql.RegisterTLSConfig() before mysql.ParseDSN() is called + // which consumes the registered tls.Config + // Fixes: https://github.com/golang-migrate/migrate/issues/411 + // + // Can't use url.Parse() since it fails to parse MySQL DSNs + // mysql.ParseDSN() also searches for "?" to find query parameters: + // https://github.com/go-sql-driver/mysql/blob/46351a8/dsn.go#L344 + if idx := strings.LastIndex(url, "?"); idx > 0 { + rawParams := url[idx+1:] + parsedParams, err := nurl.ParseQuery(rawParams) + if err != nil { + return nil, err + } - clientCert := make([]tls.Certificate, 0, 1) - if ccert, ckey := purl.Query().Get("x-tls-cert"), purl.Query().Get("x-tls-key"); ccert != "" || ckey != "" { - if ccert == "" || ckey == "" { - return nil, ErrTLSCertKeyConfig - } - certs, err := tls.LoadX509KeyPair(ccert, ckey) + ctls := parsedParams.Get("tls") + if len(ctls) > 0 { + if _, isBool := readBool(ctls); !isBool && strings.ToLower(ctls) != "skip-verify" { + rootCertPool := x509.NewCertPool() + pem, err := ioutil.ReadFile(parsedParams.Get("x-tls-ca")) if err != nil { return nil, err } - clientCert = append(clientCert, certs) - } - insecureSkipVerify := false - if len(purl.Query().Get("x-tls-insecure-skip-verify")) > 0 { - x, err := strconv.ParseBool(purl.Query().Get("x-tls-insecure-skip-verify")) + if ok := rootCertPool.AppendCertsFromPEM(pem); !ok { + return nil, ErrAppendPEM + } + + clientCert := make([]tls.Certificate, 0, 1) + if ccert, ckey := parsedParams.Get("x-tls-cert"), parsedParams.Get("x-tls-key"); ccert != "" || ckey != "" { + if ccert == "" || ckey == "" { + return nil, ErrTLSCertKeyConfig + } + certs, err := tls.LoadX509KeyPair(ccert, ckey) + if err != nil { + return nil, err + } + clientCert = append(clientCert, certs) + } + + insecureSkipVerify := false + insecureSkipVerifyStr := parsedParams.Get("x-tls-insecure-skip-verify") + if len(insecureSkipVerifyStr) > 0 { + x, err := strconv.ParseBool(insecureSkipVerifyStr) + if err != nil { + return nil, err + } + insecureSkipVerify = x + } + + err = mysql.RegisterTLSConfig(ctls, &tls.Config{ + RootCAs: rootCertPool, + Certificates: clientCert, + InsecureSkipVerify: insecureSkipVerify, + }) if err != nil { return nil, err } - insecureSkipVerify = x } - - mysql.RegisterTLSConfig(ctls, &tls.Config{ - RootCAs: rootCertPool, - Certificates: clientCert, - InsecureSkipVerify: insecureSkipVerify, - }) } } - c, err := urlToMySQLConfig(*migrate.FilterCustomQuery(purl)) + config, err := mysql.ParseDSN(strings.TrimPrefix(url, "mysql://")) + if err != nil { + return nil, err + } + + config.MultiStatements = true + + // Keep backwards compatibility from when we used net/url.Parse() to parse the DSN. + // net/url.Parse() would automatically unescape it for us. + // See: https://play.golang.org/p/q9j1io-YICQ + user, err := nurl.QueryUnescape(config.User) + if err != nil { + return nil, err + } + config.User = user + + password, err := nurl.QueryUnescape(config.Passwd) + if err != nil { + return nil, err + } + config.Passwd = password + + return config, nil +} + +func (m *Mysql) Open(url string) (database.Driver, error) { + config, err := urlToMySQLConfig(url) + if err != nil { + return nil, err + } + + customParams, err := extractCustomQueryParams(config) if err != nil { return nil, err } - db, err := sql.Open("mysql", c.FormatDSN()) + + noLockParam, noLock := customParams["x-no-lock"], false + if noLockParam != "" { + noLock, err = strconv.ParseBool(noLockParam) + if err != nil { + return nil, fmt.Errorf("could not parse x-no-lock as bool: %w", err) + } + } + + db, err := sql.Open("mysql", config.FormatDSN()) if err != nil { return nil, err } mx, err := WithInstance(db, &Config{ - DatabaseName: purl.Path, - MigrationsTable: migrationsTable, + DatabaseName: config.DBName, + MigrationsTable: customParams["x-migrations-table"], + NoLock: noLock, }) if err != nil { return nil, err @@ -196,7 +260,11 @@ func (m *Mysql) Open(url string) (database.Driver, error) { func (m *Mysql) Close() error { connErr := m.conn.Close() - dbErr := m.db.Close() + var dbErr error + if m.db != nil { + dbErr = m.db.Close() + } + if connErr != nil || dbErr != nil { return fmt.Errorf("conn: %v, db: %v", connErr, dbErr) } @@ -204,52 +272,53 @@ func (m *Mysql) Close() error { } func (m *Mysql) Lock() error { - if m.isLocked { - return database.ErrLocked - } + return database.CasRestoreOnErr(&m.isLocked, false, true, database.ErrLocked, func() error { + if m.config.NoLock { + return nil + } + aid, err := database.GenerateAdvisoryLockId( + fmt.Sprintf("%s:%s", m.config.DatabaseName, m.config.MigrationsTable)) + if err != nil { + return err + } - aid, err := database.GenerateAdvisoryLockId( - fmt.Sprintf("%s:%s", m.config.DatabaseName, m.config.MigrationsTable)) - if err != nil { - return err - } + query := "SELECT GET_LOCK(?, 10)" + var success bool + if err := m.conn.QueryRowContext(context.Background(), query, aid).Scan(&success); err != nil { + return &database.Error{OrigErr: err, Err: "try lock failed", Query: []byte(query)} + } - query := "SELECT GET_LOCK(?, 10)" - var success bool - if err := m.conn.QueryRowContext(context.Background(), query, aid).Scan(&success); err != nil { - return &database.Error{OrigErr: err, Err: "try lock failed", Query: []byte(query)} - } + if !success { + return database.ErrLocked + } - if success { - m.isLocked = true return nil - } - - return database.ErrLocked + }) } func (m *Mysql) Unlock() error { - if !m.isLocked { - return nil - } + return database.CasRestoreOnErr(&m.isLocked, true, false, database.ErrNotLocked, func() error { + if m.config.NoLock { + return nil + } - aid, err := database.GenerateAdvisoryLockId( - fmt.Sprintf("%s:%s", m.config.DatabaseName, m.config.MigrationsTable)) - if err != nil { - return err - } + aid, err := database.GenerateAdvisoryLockId( + fmt.Sprintf("%s:%s", m.config.DatabaseName, m.config.MigrationsTable)) + if err != nil { + return err + } - query := `SELECT RELEASE_LOCK(?)` - if _, err := m.conn.ExecContext(context.Background(), query, aid); err != nil { - return &database.Error{OrigErr: err, Query: []byte(query)} - } + query := `SELECT RELEASE_LOCK(?)` + if _, err := m.conn.ExecContext(context.Background(), query, aid); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } - // NOTE: RELEASE_LOCK could return NULL or (or 0 if the code is changed), - // in which case isLocked should be true until the timeout expires -- synchronizing - // these states is likely not worth trying to do; reconsider the necessity of isLocked. + // NOTE: RELEASE_LOCK could return NULL or (or 0 if the code is changed), + // in which case isLocked should be true until the timeout expires -- synchronizing + // these states is likely not worth trying to do; reconsider the necessity of isLocked. - m.isLocked = false - return nil + return nil + }) } func (m *Mysql) Run(migration io.Reader) error { @@ -274,14 +343,21 @@ func (m *Mysql) SetVersion(version int, dirty bool) error { query := "TRUNCATE `" + m.config.MigrationsTable + "`" if _, err := tx.ExecContext(context.Background(), query); err != nil { - tx.Rollback() + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } return &database.Error{OrigErr: err, Query: []byte(query)} } - if version >= 0 { + // Also re-write the schema version for nil dirty versions to prevent + // empty schema version for failed down migration on the first migration + // See: https://github.com/golang-migrate/migrate/issues/330 + if version >= 0 || (version == database.NilVersion && dirty) { query := "INSERT INTO `" + m.config.MigrationsTable + "` (version, dirty) VALUES (?, ?)" if _, err := tx.ExecContext(context.Background(), query, version, dirty); err != nil { - tx.Rollback() + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } return &database.Error{OrigErr: err, Query: []byte(query)} } } @@ -313,14 +389,18 @@ func (m *Mysql) Version() (version int, dirty bool, err error) { } } -func (m *Mysql) Drop() error { +func (m *Mysql) Drop() (err error) { // select all tables query := `SHOW TABLES LIKE '%'` tables, err := m.conn.QueryContext(context.Background(), query) if err != nil { return &database.Error{OrigErr: err, Query: []byte(query)} } - defer tables.Close() + defer func() { + if errClose := tables.Close(); errClose != nil { + err = multierror.Append(err, errClose) + } + }() // delete one table after another tableNames := make([]string, 0) @@ -333,27 +413,55 @@ func (m *Mysql) Drop() error { tableNames = append(tableNames, tableName) } } + if err := tables.Err(); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } if len(tableNames) > 0 { + // disable checking foreign key constraints until finished + query = `SET foreign_key_checks = 0` + if _, err := m.conn.ExecContext(context.Background(), query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + defer func() { + // enable foreign key checks + _, _ = m.conn.ExecContext(context.Background(), `SET foreign_key_checks = 1`) + }() + // delete one by one ... for _, t := range tableNames { - query = "DROP TABLE IF EXISTS `" + t + "` CASCADE" + query = "DROP TABLE IF EXISTS `" + t + "`" if _, err := m.conn.ExecContext(context.Background(), query); err != nil { return &database.Error{OrigErr: err, Query: []byte(query)} } } - if err := m.ensureVersionTable(); err != nil { - return err - } } return nil } -func (m *Mysql) ensureVersionTable() error { +// ensureVersionTable checks if versions table exists and, if not, creates it. +// Note that this function locks the database, which deviates from the usual +// convention of "caller locks" in the Mysql type. +func (m *Mysql) ensureVersionTable() (err error) { + if err = m.Lock(); err != nil { + return err + } + + defer func() { + if e := m.Unlock(); e != nil { + if err == nil { + err = e + } else { + err = multierror.Append(err, e) + } + } + }() + // check if migration table exists var result string - query := `SHOW TABLES LIKE "` + m.config.MigrationsTable + `"` + query := `SHOW TABLES LIKE '` + m.config.MigrationsTable + `'` if err := m.conn.QueryRowContext(context.Background(), query).Scan(&result); err != nil { if err != sql.ErrNoRows { return &database.Error{OrigErr: err, Query: []byte(query)} diff --git a/database/mysql/mysql_test.go b/database/mysql/mysql_test.go index 13a462c45..ddb18ce83 100644 --- a/database/mysql/mysql_test.go +++ b/database/mysql/mysql_test.go @@ -1,36 +1,80 @@ package mysql import ( + "context" + "crypto/ed25519" + "crypto/x509" "database/sql" sqldriver "database/sql/driver" + "encoding/pem" + "errors" "fmt" + "io/ioutil" + "log" + "math/big" + "math/rand" "net/url" + "os" + "strconv" "testing" ) import ( + "github.com/dhui/dktest" "github.com/go-sql-driver/mysql" + "github.com/stretchr/testify/assert" ) import ( + "github.com/golang-migrate/migrate/v4" dt "github.com/golang-migrate/migrate/v4/database/testing" - mt "github.com/golang-migrate/migrate/v4/testing" + "github.com/golang-migrate/migrate/v4/dktesting" + _ "github.com/golang-migrate/migrate/v4/source/file" ) -var versions = []mt.Version{ - {Image: "mysql:8", ENV: []string{"MYSQL_ROOT_PASSWORD=root", "MYSQL_DATABASE=public"}}, - {Image: "mysql:5.7", ENV: []string{"MYSQL_ROOT_PASSWORD=root", "MYSQL_DATABASE=public"}}, - {Image: "mysql:5.6", ENV: []string{"MYSQL_ROOT_PASSWORD=root", "MYSQL_DATABASE=public"}}, - {Image: "mysql:5.5", ENV: []string{"MYSQL_ROOT_PASSWORD=root", "MYSQL_DATABASE=public"}}, -} +const defaultPort = 3306 + +var ( + opts = dktest.Options{ + Env: map[string]string{"MYSQL_ROOT_PASSWORD": "root", "MYSQL_DATABASE": "public"}, + PortRequired: true, ReadyFunc: isReady, + } + optsAnsiQuotes = dktest.Options{ + Env: map[string]string{"MYSQL_ROOT_PASSWORD": "root", "MYSQL_DATABASE": "public"}, + PortRequired: true, ReadyFunc: isReady, + Cmd: []string{"--sql-mode=ANSI_QUOTES"}, + } + // Supported versions: https://www.mysql.com/support/supportedplatforms/database.html + specs = []dktesting.ContainerSpec{ + {ImageName: "mysql:5.5", Options: opts}, + {ImageName: "mysql:5.6", Options: opts}, + {ImageName: "mysql:5.7", Options: opts}, + {ImageName: "mysql:8", Options: opts}, + } + specsAnsiQuotes = []dktesting.ContainerSpec{ + {ImageName: "mysql:5.5", Options: optsAnsiQuotes}, + {ImageName: "mysql:5.6", Options: optsAnsiQuotes}, + {ImageName: "mysql:5.7", Options: optsAnsiQuotes}, + {ImageName: "mysql:8", Options: optsAnsiQuotes}, + } +) -func isReady(i mt.Instance) bool { - db, err := sql.Open("mysql", fmt.Sprintf("root:root@tcp(%v:%v)/public", i.Host(), i.Port())) +func isReady(ctx context.Context, c dktest.ContainerInfo) bool { + ip, port, err := c.Port(defaultPort) if err != nil { return false } - defer db.Close() - if err = db.Ping(); err != nil { + + db, err := sql.Open("mysql", fmt.Sprintf("root:root@tcp(%v:%v)/public", ip, port)) + if err != nil { + return false + } + defer func() { + if err := db.Close(); err != nil { + log.Println("close error:", err) + } + }() + if err = db.PingContext(ctx); err != nil { switch err { case sqldriver.ErrBadConn, mysql.ErrInvalidConn: return false @@ -46,63 +90,293 @@ func isReady(i mt.Instance) bool { func Test(t *testing.T) { // mysql.SetLogger(mysql.Logger(log.New(ioutil.Discard, "", log.Ltime))) - mt.ParallelTest(t, versions, isReady, - func(t *testing.T, i mt.Instance) { - p := &Mysql{} - addr := fmt.Sprintf("mysql://root:root@tcp(%v:%v)/public", i.Host(), i.Port()) - d, err := p.Open(addr) - if err != nil { - t.Fatalf("%v", err) + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } + + addr := fmt.Sprintf("mysql://root:root@tcp(%v:%v)/public", ip, port) + p := &Mysql{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) } - defer d.Close() - dt.Test(t, d, []byte("SELECT 1")) + }() + dt.Test(t, d, []byte("SELECT 1")) + + // check ensureVersionTable + if err := d.(*Mysql).ensureVersionTable(); err != nil { + t.Fatal(err) + } + // check again + if err := d.(*Mysql).ensureVersionTable(); err != nil { + t.Fatal(err) + } + }) +} + +func TestMigrate(t *testing.T) { + // mysql.SetLogger(mysql.Logger(log.New(ioutil.Discard, "", log.Ltime))) - // check ensureVersionTable - if err := d.(*Mysql).ensureVersionTable(); err != nil { - t.Fatal(err) + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } + + addr := fmt.Sprintf("mysql://root:root@tcp(%v:%v)/public", ip, port) + p := &Mysql{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) } - // check again - if err := d.(*Mysql).ensureVersionTable(); err != nil { - t.Fatal(err) + }() + + m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "public", d) + if err != nil { + t.Fatal(err) + } + dt.TestMigrate(t, m) + + // check ensureVersionTable + if err := d.(*Mysql).ensureVersionTable(); err != nil { + t.Fatal(err) + } + // check again + if err := d.(*Mysql).ensureVersionTable(); err != nil { + t.Fatal(err) + } + }) +} + +func TestMigrateAnsiQuotes(t *testing.T) { + // mysql.SetLogger(mysql.Logger(log.New(ioutil.Discard, "", log.Ltime))) + + dktesting.ParallelTest(t, specsAnsiQuotes, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } + + addr := fmt.Sprintf("mysql://root:root@tcp(%v:%v)/public", ip, port) + p := &Mysql{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) } - }) + }() + + m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "public", d) + if err != nil { + t.Fatal(err) + } + dt.TestMigrate(t, m) + + // check ensureVersionTable + if err := d.(*Mysql).ensureVersionTable(); err != nil { + t.Fatal(err) + } + // check again + if err := d.(*Mysql).ensureVersionTable(); err != nil { + t.Fatal(err) + } + }) } func TestLockWorks(t *testing.T) { - mt.ParallelTest(t, versions, isReady, - func(t *testing.T, i mt.Instance) { - p := &Mysql{} - addr := fmt.Sprintf("mysql://root:root@tcp(%v:%v)/public", i.Host(), i.Port()) - d, err := p.Open(addr) - if err != nil { - t.Fatalf("%v", err) - } - dt.Test(t, d, []byte("SELECT 1")) + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } - ms := d.(*Mysql) + addr := fmt.Sprintf("mysql://root:root@tcp(%v:%v)/public", ip, port) + p := &Mysql{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + dt.Test(t, d, []byte("SELECT 1")) - err = ms.Lock() - if err != nil { - t.Fatal(err) - } - err = ms.Unlock() - if err != nil { - t.Fatal(err) - } + ms := d.(*Mysql) - // make sure the 2nd lock works (RELEASE_LOCK is very finicky) - err = ms.Lock() - if err != nil { - t.Fatal(err) - } - err = ms.Unlock() - if err != nil { - t.Fatal(err) + err = ms.Lock() + if err != nil { + t.Fatal(err) + } + err = ms.Unlock() + if err != nil { + t.Fatal(err) + } + + // make sure the 2nd lock works (RELEASE_LOCK is very finicky) + err = ms.Lock() + if err != nil { + t.Fatal(err) + } + err = ms.Unlock() + if err != nil { + t.Fatal(err) + } + }) +} + +func TestNoLockParamValidation(t *testing.T) { + ip := "127.0.0.1" + port := 3306 + addr := fmt.Sprintf("mysql://root:root@tcp(%v:%v)/public", ip, port) + p := &Mysql{} + _, err := p.Open(addr + "?x-no-lock=not-a-bool") + if !errors.Is(err, strconv.ErrSyntax) { + t.Fatal("Expected syntax error when passing a non-bool as x-no-lock parameter") + } +} + +func TestNoLockWorks(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } + + addr := fmt.Sprintf("mysql://root:root@tcp(%v:%v)/public", ip, port) + p := &Mysql{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + + lock := d.(*Mysql) + + p = &Mysql{} + d, err = p.Open(addr + "?x-no-lock=true") + if err != nil { + t.Fatal(err) + } + + noLock := d.(*Mysql) + + // Should be possible to take real lock and no-lock at the same time + if err = lock.Lock(); err != nil { + t.Fatal(err) + } + if err = noLock.Lock(); err != nil { + t.Fatal(err) + } + if err = lock.Unlock(); err != nil { + t.Fatal(err) + } + if err = noLock.Unlock(); err != nil { + t.Fatal(err) + } + }) +} + +func TestExtractCustomQueryParams(t *testing.T) { + testcases := []struct { + name string + config *mysql.Config + expectedParams map[string]string + expectedCustomParams map[string]string + expectedErr error + }{ + {name: "nil config", expectedErr: ErrNilConfig}, + { + name: "no params", + config: mysql.NewConfig(), + expectedCustomParams: map[string]string{}, + }, + { + name: "no custom params", + config: &mysql.Config{Params: map[string]string{"hello": "world"}}, + expectedParams: map[string]string{"hello": "world"}, + expectedCustomParams: map[string]string{}, + }, + { + name: "one param, one custom param", + config: &mysql.Config{ + Params: map[string]string{"hello": "world", "x-foo": "bar"}, + }, + expectedParams: map[string]string{"hello": "world"}, + expectedCustomParams: map[string]string{"x-foo": "bar"}, + }, + { + name: "multiple params, multiple custom params", + config: &mysql.Config{ + Params: map[string]string{ + "hello": "world", + "x-foo": "bar", + "dead": "beef", + "x-cat": "hat", + }, + }, + expectedParams: map[string]string{"hello": "world", "dead": "beef"}, + expectedCustomParams: map[string]string{"x-foo": "bar", "x-cat": "hat"}, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + customParams, err := extractCustomQueryParams(tc.config) + if tc.config != nil { + assert.Equal(t, tc.expectedParams, tc.config.Params, + "Expected config params have custom params properly removed") } + assert.Equal(t, tc.expectedErr, err, "Expected errors to match") + assert.Equal(t, tc.expectedCustomParams, customParams, + "Expected custom params to be properly extracted") }) + } +} + +func createTmpCert(t *testing.T) string { + tmpCertFile, err := ioutil.TempFile("", "migrate_test_cert") + if err != nil { + t.Fatal("Failed to create temp cert file:", err) + } + t.Cleanup(func() { + if err := os.Remove(tmpCertFile.Name()); err != nil { + t.Log("Failed to cleanup temp cert file:", err) + } + }) + + r := rand.New(rand.NewSource(0)) + pub, priv, err := ed25519.GenerateKey(r) + if err != nil { + t.Fatal("Failed to generate ed25519 key for temp cert file:", err) + } + tmpl := x509.Certificate{ + SerialNumber: big.NewInt(0), + } + derBytes, err := x509.CreateCertificate(r, &tmpl, &tmpl, pub, priv) + if err != nil { + t.Fatal("Failed to generate temp cert file:", err) + } + if err := pem.Encode(tmpCertFile, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil { + t.Fatal("Failed to encode ") + } + if err := tmpCertFile.Close(); err != nil { + t.Fatal("Failed to close temp cert file:", err) + } + return tmpCertFile.Name() } func TestURLToMySQLConfig(t *testing.T) { + tmpCertFilename := createTmpCert(t) + tmpCertFilenameEscaped := url.PathEscape(tmpCertFilename) + testcases := []struct { name string urlStr string @@ -133,22 +407,19 @@ func TestURLToMySQLConfig(t *testing.T) { {name: "user/password - password with encoded @", urlStr: "mysql://username:password%40@tcp(127.0.0.1:3306)/myDB?multiStatements=true", expectedDSN: "username:password@@tcp(127.0.0.1:3306)/myDB?multiStatements=true"}, + {name: "custom tls", + urlStr: "mysql://username:password@tcp(127.0.0.1:3306)/myDB?multiStatements=true&tls=custom&x-tls-ca=" + tmpCertFilenameEscaped, + expectedDSN: "username:password@tcp(127.0.0.1:3306)/myDB?multiStatements=true&tls=custom&x-tls-ca=" + tmpCertFilenameEscaped}, } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - u, err := url.Parse(tc.urlStr) + config, err := urlToMySQLConfig(tc.urlStr) if err != nil { t.Fatal("Failed to parse url string:", tc.urlStr, "error:", err) } - if config, err := urlToMySQLConfig(*u); err == nil { - dsn := config.FormatDSN() - if dsn != tc.expectedDSN { - t.Error("Got unexpected DSN:", dsn, "!=", tc.expectedDSN) - } - } else { - if tc.expectedDSN != "" { - t.Error("Got unexpected error:", err, "urlStr:", tc.urlStr) - } + dsn := config.FormatDSN() + if dsn != tc.expectedDSN { + t.Error("Got unexpected DSN:", dsn, "!=", tc.expectedDSN) } }) } diff --git a/database/neo4j/README.md b/database/neo4j/README.md index e69de29bb..60b6ab507 100644 --- a/database/neo4j/README.md +++ b/database/neo4j/README.md @@ -0,0 +1,20 @@ +# neo4j +The Neo4j driver (bolt) does not natively support executing multiple statements in a single query. To allow for multiple statements in a single migration, you can use the `x-multi-statement` param. +This mode splits the migration text into separately-executed statements by a semi-colon `;`. Thus `x-multi-statement` cannot be used when a statement in the migration contains a string with a semi-colon. +The queries **should** run in a single transaction, so partial migrations should not be a concern, but this is untested. + + +`neo4j://user:password@host:port/` + +| URL Query | WithInstance Config | Description | +|------------|---------------------|-------------| +| `x-multi-statement` | `MultiStatement` | Enable multiple statements to be ran in a single migration (See note above) | +| `user` | Contained within `AuthConfig` | The user to sign in as | +| `password` | Contained within `AuthConfig` | The user's password | +| `host` | | The host to connect to. Values that start with / are for unix domain sockets. (default is localhost) | +| `port` | | The port to bind to. (default is 7687) | +| | `MigrationsLabel` | Name of the migrations node label | + +## Supported versions + +Only Neo4j v3.5+ is [supported](https://github.com/neo4j/neo4j-go-driver/issues/64#issuecomment-625133600) diff --git a/database/neo4j/TUTORIAL.md b/database/neo4j/TUTORIAL.md new file mode 100644 index 000000000..14bb1986c --- /dev/null +++ b/database/neo4j/TUTORIAL.md @@ -0,0 +1,97 @@ +## Create migrations +Let's create nodes called `Users`: +``` +migrate create -ext cypher -dir db/migrations -seq create_user_nodes +``` +If there were no errors, we should have two files available under `db/migrations` folder: +- 000001_create_user_nodes.down.cypher +- 000001_create_user_nodes.up.cypher + +Note the `cypher` extension that we provided. + +In the `.up.cypher` file let's create the table: +``` +CREATE (u1:User {name: "Peter"}) +CREATE (u2:User {name: "Paul"}) +CREATE (u3:User {name: "Mary"}) +``` +And in the `.down.sql` let's delete it: +``` +MATCH (u:User) WHERE u.name IN ["Peter", "Paul", "Mary"] DELETE u +``` +Ideally your migrations should be idempotent. You can read more about idempotency in [getting started](GETTING_STARTED.md#create-migrations) + +## Run migrations +``` +migrate -database ${NEO4J_URL} -path db/migrations up +``` +Let's check if the table was created properly by running `bin/cypher-shell -u neo4j -p password`, then `neo4j> MATCH (u:User)` +The output you are supposed to see: +``` ++-----------------------------------------------------------------+ +| u | ++-----------------------------------------------------------------+ +| (:User {name: "Peter") | +| (:User {name: "Paul") | +| (:User {name: "Mary") | ++-----------------------------------------------------------------+ +``` +Great! Now let's check if running reverse migration also works: +``` +migrate -database ${NEO4J_URL} -path db/migrations down +``` +Make sure to check if your database changed as expected in this case as well. + +## Database transactions + +To show database transactions usage, let's create another set of migrations by running: +``` +migrate create -ext cypher -dir db/migrations -seq add_mood_to_users +``` +Again, it should create for us two migrations files: +- 000002_add_mood_to_users.down.cypher +- 000002_add_mood_to_users.up.cypher + +In Neo4j, when we want our queries to be done in a transaction, we need to wrap it with `:BEGIN` and `:COMMIT` commands. +Migration up: +``` +:BEGIN + +MATCH (u:User) +SET u.mood = "Cheery" + +:COMMIT +``` +Migration down: +``` +:BEGIN + +MATCH (u:User) +SET u.mood = null + +:COMMIT +``` + +## Optional: Run migrations within your Go app +Here is a very simple app running migrations for the above configuration: +``` +import ( + "log" + + "github.com/golang-migrate/migrate/v4" + _ "github.com/golang-migrate/migrate/v4/database/neo4j" + _ "github.com/golang-migrate/migrate/v4/source/file" +) + +func main() { + m, err := migrate.New( + "file://db/migrations", + "neo4j://neo4j:password@localhost:7687/") + if err != nil { + log.Fatal(err) + } + if err := m.Up(); err != nil { + log.Fatal(err) + } +} +``` \ No newline at end of file diff --git a/database/neo4j/examples/migrations/1578421040_create_movies_constraint.down.cypher b/database/neo4j/examples/migrations/1578421040_create_movies_constraint.down.cypher new file mode 100644 index 000000000..8e65d5aef --- /dev/null +++ b/database/neo4j/examples/migrations/1578421040_create_movies_constraint.down.cypher @@ -0,0 +1 @@ +DROP CONSTRAINT ON (m:Movie) ASSERT m.Name IS UNIQUE \ No newline at end of file diff --git a/database/neo4j/examples/migrations/1578421040_create_movies_constraint.up.cypher b/database/neo4j/examples/migrations/1578421040_create_movies_constraint.up.cypher new file mode 100644 index 000000000..4ca81fed1 --- /dev/null +++ b/database/neo4j/examples/migrations/1578421040_create_movies_constraint.up.cypher @@ -0,0 +1 @@ +CREATE CONSTRAINT ON (m:Movie) ASSERT m.Name IS UNIQUE \ No newline at end of file diff --git a/database/neo4j/examples/migrations/1578421725_create_movies.down.cypher b/database/neo4j/examples/migrations/1578421725_create_movies.down.cypher new file mode 100644 index 000000000..110dd68de --- /dev/null +++ b/database/neo4j/examples/migrations/1578421725_create_movies.down.cypher @@ -0,0 +1,2 @@ +MATCH (m:Movie) +DELETE m \ No newline at end of file diff --git a/database/neo4j/examples/migrations/1578421725_create_movies.up.cypher b/database/neo4j/examples/migrations/1578421725_create_movies.up.cypher new file mode 100644 index 000000000..5283d85ac --- /dev/null +++ b/database/neo4j/examples/migrations/1578421725_create_movies.up.cypher @@ -0,0 +1,2 @@ +CREATE (:Movie {name: "Footloose"}) +CREATE (:Movie {name: "Ghost"}) \ No newline at end of file diff --git a/database/neo4j/examples/migrations/1578421726_multistatement_test.up.cypher b/database/neo4j/examples/migrations/1578421726_multistatement_test.up.cypher new file mode 100644 index 000000000..2482a0998 --- /dev/null +++ b/database/neo4j/examples/migrations/1578421726_multistatement_test.up.cypher @@ -0,0 +1,3 @@ +CREATE (:Movie {name: "Hollow Man"}); +CREATE (:Movie {name: "Mystic River"}); +;;; \ No newline at end of file diff --git a/database/neo4j/neo4j.go b/database/neo4j/neo4j.go new file mode 100644 index 000000000..7796412af --- /dev/null +++ b/database/neo4j/neo4j.go @@ -0,0 +1,304 @@ +package neo4j + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + neturl "net/url" + "strconv" + "sync/atomic" + + "github.com/golang-migrate/migrate/v4/database" + "github.com/golang-migrate/migrate/v4/database/multistmt" + "github.com/hashicorp/go-multierror" + "github.com/neo4j/neo4j-go-driver/neo4j" +) + +func init() { + db := Neo4j{} + database.Register("neo4j", &db) +} + +const DefaultMigrationsLabel = "SchemaMigration" + +var ( + StatementSeparator = []byte(";") + DefaultMultiStatementMaxSize = 10 * 1 << 20 // 10 MB +) + +var ( + ErrNilConfig = fmt.Errorf("no config") +) + +type Config struct { + MigrationsLabel string + MultiStatement bool + MultiStatementMaxSize int +} + +type Neo4j struct { + driver neo4j.Driver + lock uint32 + + // Open and WithInstance need to guarantee that config is never nil + config *Config +} + +func WithInstance(driver neo4j.Driver, config *Config) (database.Driver, error) { + if config == nil { + return nil, ErrNilConfig + } + + nDriver := &Neo4j{ + driver: driver, + config: config, + } + + if err := nDriver.ensureVersionConstraint(); err != nil { + return nil, err + } + + return nDriver, nil +} + +func (n *Neo4j) Open(url string) (database.Driver, error) { + uri, err := neturl.Parse(url) + if err != nil { + return nil, err + } + password, _ := uri.User.Password() + authToken := neo4j.BasicAuth(uri.User.Username(), password, "") + uri.User = nil + uri.Scheme = "bolt" + msQuery := uri.Query().Get("x-multi-statement") + + // Whether to turn on/off TLS encryption. + tlsEncrypted := uri.Query().Get("x-tls-encrypted") + multi := false + encrypted := false + if msQuery != "" { + multi, err = strconv.ParseBool(uri.Query().Get("x-multi-statement")) + if err != nil { + return nil, err + } + } + + if tlsEncrypted != "" { + encrypted, err = strconv.ParseBool(tlsEncrypted) + if err != nil { + return nil, err + } + } + + multiStatementMaxSize := DefaultMultiStatementMaxSize + if s := uri.Query().Get("x-multi-statement-max-size"); s != "" { + multiStatementMaxSize, err = strconv.Atoi(s) + if err != nil { + return nil, err + } + } + + uri.RawQuery = "" + + driver, err := neo4j.NewDriver(uri.String(), authToken, func(config *neo4j.Config) { + config.Encrypted = encrypted + }) + if err != nil { + return nil, err + } + + return WithInstance(driver, &Config{ + MigrationsLabel: DefaultMigrationsLabel, + MultiStatement: multi, + MultiStatementMaxSize: multiStatementMaxSize, + }) +} + +func (n *Neo4j) Close() error { + return n.driver.Close() +} + +// local locking in order to pass tests, Neo doesn't support database locking +func (n *Neo4j) Lock() error { + if !atomic.CompareAndSwapUint32(&n.lock, 0, 1) { + return database.ErrLocked + } + + return nil +} + +func (n *Neo4j) Unlock() error { + if !atomic.CompareAndSwapUint32(&n.lock, 1, 0) { + return database.ErrNotLocked + } + return nil +} + +func (n *Neo4j) Run(migration io.Reader) (err error) { + session, err := n.driver.Session(neo4j.AccessModeWrite) + if err != nil { + return err + } + defer func() { + if cerr := session.Close(); cerr != nil { + err = multierror.Append(err, cerr) + } + }() + + if n.config.MultiStatement { + _, err = session.WriteTransaction(func(transaction neo4j.Transaction) (interface{}, error) { + var stmtRunErr error + if err := multistmt.Parse(migration, StatementSeparator, n.config.MultiStatementMaxSize, func(stmt []byte) bool { + trimStmt := bytes.TrimSpace(stmt) + if len(trimStmt) == 0 { + return true + } + trimStmt = bytes.TrimSuffix(trimStmt, StatementSeparator) + if len(trimStmt) == 0 { + return true + } + + result, err := transaction.Run(string(trimStmt), nil) + if _, err := neo4j.Collect(result, err); err != nil { + stmtRunErr = err + return false + } + return true + }); err != nil { + return nil, err + } + return nil, stmtRunErr + }) + return err + } + + body, err := ioutil.ReadAll(migration) + if err != nil { + return err + } + + _, err = neo4j.Collect(session.Run(string(body[:]), nil)) + return err +} + +func (n *Neo4j) SetVersion(version int, dirty bool) (err error) { + session, err := n.driver.Session(neo4j.AccessModeWrite) + if err != nil { + return err + } + defer func() { + if cerr := session.Close(); cerr != nil { + err = multierror.Append(err, cerr) + } + }() + + query := fmt.Sprintf("MERGE (sm:%s {version: $version}) SET sm.dirty = $dirty, sm.ts = datetime()", + n.config.MigrationsLabel) + _, err = neo4j.Collect(session.Run(query, map[string]interface{}{"version": version, "dirty": dirty})) + if err != nil { + return err + } + return nil +} + +type MigrationRecord struct { + Version int + Dirty bool +} + +func (n *Neo4j) Version() (version int, dirty bool, err error) { + session, err := n.driver.Session(neo4j.AccessModeRead) + if err != nil { + return database.NilVersion, false, err + } + defer func() { + if cerr := session.Close(); cerr != nil { + err = multierror.Append(err, cerr) + } + }() + + query := fmt.Sprintf(`MATCH (sm:%s) RETURN sm.version AS version, sm.dirty AS dirty +ORDER BY COALESCE(sm.ts, datetime({year: 0})) DESC, sm.version DESC LIMIT 1`, + n.config.MigrationsLabel) + result, err := session.ReadTransaction(func(transaction neo4j.Transaction) (interface{}, error) { + result, err := transaction.Run(query, nil) + if err != nil { + return nil, err + } + if result.Next() { + record := result.Record() + mr := MigrationRecord{} + versionResult, ok := record.Get("version") + if !ok { + mr.Version = database.NilVersion + } else { + mr.Version = int(versionResult.(int64)) + } + + dirtyResult, ok := record.Get("dirty") + if ok { + mr.Dirty = dirtyResult.(bool) + } + + return mr, nil + } + return nil, result.Err() + }) + if err != nil { + return database.NilVersion, false, err + } + if result == nil { + return database.NilVersion, false, err + } + mr := result.(MigrationRecord) + return mr.Version, mr.Dirty, err +} + +func (n *Neo4j) Drop() (err error) { + session, err := n.driver.Session(neo4j.AccessModeWrite) + if err != nil { + return err + } + defer func() { + if cerr := session.Close(); cerr != nil { + err = multierror.Append(err, cerr) + } + }() + + if _, err := neo4j.Collect(session.Run("MATCH (n) DETACH DELETE n", nil)); err != nil { + return err + } + return nil +} + +func (n *Neo4j) ensureVersionConstraint() (err error) { + session, err := n.driver.Session(neo4j.AccessModeWrite) + if err != nil { + return err + } + defer func() { + if cerr := session.Close(); cerr != nil { + err = multierror.Append(err, cerr) + } + }() + + /** + Get constraint and check to avoid error duplicate + using db.labels() to support Neo4j 3 and 4. + Neo4J 3 doesn't support db.constraints() YIELD name + */ + res, err := neo4j.Collect(session.Run(fmt.Sprintf("CALL db.labels() YIELD label WHERE label=\"%s\" RETURN label", n.config.MigrationsLabel), nil)) + if err != nil { + return err + } + if len(res) == 1 { + return nil + } + + query := fmt.Sprintf("CREATE CONSTRAINT ON (a:%s) ASSERT a.version IS UNIQUE", n.config.MigrationsLabel) + if _, err := neo4j.Collect(session.Run(query, nil)); err != nil { + return err + } + return nil +} diff --git a/database/neo4j/neo4j_test.go b/database/neo4j/neo4j_test.go new file mode 100644 index 000000000..c8e914525 --- /dev/null +++ b/database/neo4j/neo4j_test.go @@ -0,0 +1,138 @@ +package neo4j + +import ( + "bytes" + "context" + "fmt" + "log" + "testing" + + "github.com/dhui/dktest" + "github.com/neo4j/neo4j-go-driver/neo4j" + + "github.com/golang-migrate/migrate/v4" + dt "github.com/golang-migrate/migrate/v4/database/testing" + "github.com/golang-migrate/migrate/v4/dktesting" + _ "github.com/golang-migrate/migrate/v4/source/file" +) + +var ( + opts = dktest.Options{PortRequired: true, ReadyFunc: isReady, + Env: map[string]string{"NEO4J_AUTH": "neo4j/migratetest", "NEO4J_ACCEPT_LICENSE_AGREEMENT": "yes"}} + specs = []dktesting.ContainerSpec{ + {ImageName: "neo4j:4.0", Options: opts}, + {ImageName: "neo4j:4.0-enterprise", Options: opts}, + {ImageName: "neo4j:3.5", Options: opts}, + {ImageName: "neo4j:3.5-enterprise", Options: opts}, + } +) + +func neoConnectionString(host, port string) string { + return fmt.Sprintf("bolt://neo4j:migratetest@%s:%s", host, port) +} + +func isReady(ctx context.Context, c dktest.ContainerInfo) bool { + ip, port, err := c.Port(7687) + if err != nil { + return false + } + + driver, err := neo4j.NewDriver( + neoConnectionString(ip, port), + neo4j.BasicAuth("neo4j", "migratetest", ""), + func(config *neo4j.Config) { + config.Encrypted = false + }) + if err != nil { + return false + } + defer func() { + if err := driver.Close(); err != nil { + log.Println("close error:", err) + } + }() + session, err := driver.Session(neo4j.AccessModeRead) + if err != nil { + return false + } + result, err := session.Run("RETURN 1", nil) + if err != nil { + return false + } else if result.Err() != nil { + return false + } + + return true +} + +func Test(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(7687) + if err != nil { + t.Fatal(err) + } + + n := &Neo4j{} + d, err := n.Open(neoConnectionString(ip, port)) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + dt.Test(t, d, []byte("MATCH (a) RETURN a")) + }) +} + +func TestMigrate(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(7687) + if err != nil { + t.Fatal(err) + } + + n := &Neo4j{} + neoUrl := neoConnectionString(ip, port) + "/?x-multi-statement=true" + d, err := n.Open(neoUrl) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "neo4j", d) + if err != nil { + t.Fatal(err) + } + dt.TestMigrate(t, m) + }) +} + +func TestMalformed(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(7687) + if err != nil { + t.Fatal(err) + } + + n := &Neo4j{} + d, err := n.Open(neoConnectionString(ip, port)) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + + migration := bytes.NewReader([]byte("CREATE (a {qid: 1) RETURN a")) + if err := d.Run(migration); err == nil { + t.Fatal("expected failure for malformed migration") + } + }) +} diff --git a/database/parse_test.go b/database/parse_test.go index b363357a8..3709a6796 100644 --- a/database/parse_test.go +++ b/database/parse_test.go @@ -9,11 +9,12 @@ import ( const reservedChars = "!#$%&'()*+,/:;=?@[]" +const baseUsername = "username" + // TestUserUnencodedReservedURLChars documents the behavior of using unencoded reserved characters in usernames with // net/url Parse() func TestUserUnencodedReservedURLChars(t *testing.T) { scheme := "database://" - baseUsername := "username" urlSuffix := "password@localhost:12345/myDB?someParam=true" urlSuffixAndSep := ":" + urlSuffix @@ -46,7 +47,7 @@ func TestUserUnencodedReservedURLChars(t *testing.T) { encodedURL: scheme + baseUsername + "," + urlSuffixAndSep}, {char: "/", parses: true, expectedUsername: "", encodedURL: scheme + baseUsername + "/" + urlSuffixAndSep}, - {char: ":", parses: true, expectedUsername: "username", + {char: ":", parses: true, expectedUsername: baseUsername, encodedURL: scheme + baseUsername + ":%3A" + urlSuffix}, {char: ";", parses: true, expectedUsername: "username;", encodedURL: scheme + baseUsername + ";" + urlSuffixAndSep}, @@ -98,7 +99,6 @@ func TestUserUnencodedReservedURLChars(t *testing.T) { func TestUserEncodedReservedURLChars(t *testing.T) { scheme := "database://" - baseUsername := "username" urlSuffix := "password@localhost:12345/myDB?someParam=true" urlSuffixAndSep := ":" + urlSuffix @@ -125,7 +125,7 @@ func TestUserEncodedReservedURLChars(t *testing.T) { // TestPasswordUnencodedReservedURLChars documents the behavior of using unencoded reserved characters in passwords // with net/url Parse() func TestPasswordUnencodedReservedURLChars(t *testing.T) { - username := "username" + username := baseUsername schemeAndUsernameAndSep := "database://" + username + ":" basePassword := "password" urlSuffixAndSep := "@localhost:12345/myDB?someParam=true" @@ -212,7 +212,7 @@ func TestPasswordUnencodedReservedURLChars(t *testing.T) { } func TestPasswordEncodedReservedURLChars(t *testing.T) { - username := "username" + username := baseUsername schemeAndUsernameAndSep := "database://" + username + ":" basePassword := "password" urlSuffixAndSep := "@localhost:12345/myDB?someParam=true" diff --git a/database/pgx/README.md b/database/pgx/README.md new file mode 100644 index 000000000..dfe150a72 --- /dev/null +++ b/database/pgx/README.md @@ -0,0 +1,39 @@ +# pgx + +`pgx://user:password@host:port/dbname?query` + +| URL Query | WithInstance Config | Description | +|------------|---------------------|-------------| +| `x-migrations-table` | `MigrationsTable` | Name of the migrations table | +| `x-migrations-table-quoted` | `MigrationsTableQuoted` | By default, migrate quotes the migration table for SQL injection safety reasons. This option disable quoting and naively checks that you have quoted the migration table name. e.g. `"my_schema"."schema_migrations"` | +| `x-statement-timeout` | `StatementTimeout` | Abort any statement that takes more than the specified number of milliseconds | +| `x-multi-statement` | `MultiStatementEnabled` | Enable multi-statement execution (default: false) | +| `x-multi-statement-max-size` | `MultiStatementMaxSize` | Maximum size of single statement in bytes (default: 10MB) | +| `dbname` | `DatabaseName` | The name of the database to connect to | +| `search_path` | | This variable specifies the order in which schemas are searched when an object is referenced by a simple name with no schema specified. | +| `user` | | The user to sign in as | +| `password` | | The user's password | +| `host` | | The host to connect to. Values that start with / are for unix domain sockets. (default is localhost) | +| `port` | | The port to bind to. (default is 5432) | +| `fallback_application_name` | | An application_name to fall back to if one isn't provided. | +| `connect_timeout` | | Maximum wait for connection, in seconds. Zero or not specified means wait indefinitely. | +| `sslcert` | | Cert file location. The file must contain PEM encoded data. | +| `sslkey` | | Key file location. The file must contain PEM encoded data. | +| `sslrootcert` | | The location of the root certificate file. The file must contain PEM encoded data. | +| `sslmode` | | Whether or not to use SSL (disable\|require\|verify-ca\|verify-full) | + + +## Upgrading from v1 + +1. Write down the current migration version from schema_migrations +1. `DROP TABLE schema_migrations` +2. Wrap your existing migrations in transactions ([BEGIN/COMMIT](https://www.postgresql.org/docs/current/static/transaction-iso.html)) if you use multiple statements within one migration. +3. Download and install the latest migrate version. +4. Force the current migration version with `migrate force `. + +## Multi-statement mode + +In PostgreSQL running multiple SQL statements in one `Exec` executes them inside a transaction. Sometimes this +behavior is not desirable because some statements can be only run outside of transaction (e.g. +`CREATE INDEX CONCURRENTLY`). If you want to use `CREATE INDEX CONCURRENTLY` without activating multi-statement mode +you have to put such statements in a separate migration files. diff --git a/database/pgx/examples/migrations/1085649617_create_users_table.down.sql b/database/pgx/examples/migrations/1085649617_create_users_table.down.sql new file mode 100644 index 000000000..c99ddcdc8 --- /dev/null +++ b/database/pgx/examples/migrations/1085649617_create_users_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS users; diff --git a/database/pgx/examples/migrations/1085649617_create_users_table.up.sql b/database/pgx/examples/migrations/1085649617_create_users_table.up.sql new file mode 100644 index 000000000..92897dcab --- /dev/null +++ b/database/pgx/examples/migrations/1085649617_create_users_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE users ( + user_id integer unique, + name varchar(40), + email varchar(40) +); diff --git a/database/pgx/examples/migrations/1185749658_add_city_to_users.down.sql b/database/pgx/examples/migrations/1185749658_add_city_to_users.down.sql new file mode 100644 index 000000000..940c60712 --- /dev/null +++ b/database/pgx/examples/migrations/1185749658_add_city_to_users.down.sql @@ -0,0 +1 @@ +ALTER TABLE users DROP COLUMN IF EXISTS city; diff --git a/database/pgx/examples/migrations/1185749658_add_city_to_users.up.sql b/database/pgx/examples/migrations/1185749658_add_city_to_users.up.sql new file mode 100644 index 000000000..67823edc9 --- /dev/null +++ b/database/pgx/examples/migrations/1185749658_add_city_to_users.up.sql @@ -0,0 +1,3 @@ +ALTER TABLE users ADD COLUMN city varchar(100); + + diff --git a/database/pgx/examples/migrations/1285849751_add_index_on_user_emails.down.sql b/database/pgx/examples/migrations/1285849751_add_index_on_user_emails.down.sql new file mode 100644 index 000000000..3e87dd229 --- /dev/null +++ b/database/pgx/examples/migrations/1285849751_add_index_on_user_emails.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS users_email_index; diff --git a/database/pgx/examples/migrations/1285849751_add_index_on_user_emails.up.sql b/database/pgx/examples/migrations/1285849751_add_index_on_user_emails.up.sql new file mode 100644 index 000000000..fbeb4ab4e --- /dev/null +++ b/database/pgx/examples/migrations/1285849751_add_index_on_user_emails.up.sql @@ -0,0 +1,3 @@ +CREATE UNIQUE INDEX CONCURRENTLY users_email_index ON users (email); + +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/pgx/examples/migrations/1385949617_create_books_table.down.sql b/database/pgx/examples/migrations/1385949617_create_books_table.down.sql new file mode 100644 index 000000000..1a0b1a214 --- /dev/null +++ b/database/pgx/examples/migrations/1385949617_create_books_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS books; diff --git a/database/pgx/examples/migrations/1385949617_create_books_table.up.sql b/database/pgx/examples/migrations/1385949617_create_books_table.up.sql new file mode 100644 index 000000000..f1503b518 --- /dev/null +++ b/database/pgx/examples/migrations/1385949617_create_books_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE books ( + user_id integer, + name varchar(40), + author varchar(40) +); diff --git a/database/pgx/examples/migrations/1485949617_create_movies_table.down.sql b/database/pgx/examples/migrations/1485949617_create_movies_table.down.sql new file mode 100644 index 000000000..3a5187689 --- /dev/null +++ b/database/pgx/examples/migrations/1485949617_create_movies_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS movies; diff --git a/database/pgx/examples/migrations/1485949617_create_movies_table.up.sql b/database/pgx/examples/migrations/1485949617_create_movies_table.up.sql new file mode 100644 index 000000000..f0ef5943b --- /dev/null +++ b/database/pgx/examples/migrations/1485949617_create_movies_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE movies ( + user_id integer, + name varchar(40), + director varchar(40) +); diff --git a/database/pgx/examples/migrations/1585849751_just_a_comment.up.sql b/database/pgx/examples/migrations/1585849751_just_a_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/pgx/examples/migrations/1585849751_just_a_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/pgx/examples/migrations/1685849751_another_comment.up.sql b/database/pgx/examples/migrations/1685849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/pgx/examples/migrations/1685849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/pgx/examples/migrations/1785849751_another_comment.up.sql b/database/pgx/examples/migrations/1785849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/pgx/examples/migrations/1785849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/pgx/examples/migrations/1885849751_another_comment.up.sql b/database/pgx/examples/migrations/1885849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/pgx/examples/migrations/1885849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/pgx/pgx.go b/database/pgx/pgx.go new file mode 100644 index 000000000..2fffe54df --- /dev/null +++ b/database/pgx/pgx.go @@ -0,0 +1,487 @@ +//go:build go1.9 +// +build go1.9 + +package pgx + +import ( + "context" + "database/sql" + "fmt" + "go.uber.org/atomic" + "io" + "io/ioutil" + nurl "net/url" + "regexp" + "strconv" + "strings" + "time" + + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database" + "github.com/golang-migrate/migrate/v4/database/multistmt" + multierror "github.com/hashicorp/go-multierror" + "github.com/jackc/pgconn" + "github.com/jackc/pgerrcode" + _ "github.com/jackc/pgx/v4/stdlib" +) + +func init() { + db := Postgres{} + database.Register("pgx", &db) +} + +var ( + multiStmtDelimiter = []byte(";") + + DefaultMigrationsTable = "schema_migrations" + DefaultMultiStatementMaxSize = 10 * 1 << 20 // 10 MB +) + +var ( + ErrNilConfig = fmt.Errorf("no config") + ErrNoDatabaseName = fmt.Errorf("no database name") + ErrNoSchema = fmt.Errorf("no schema") + ErrDatabaseDirty = fmt.Errorf("database is dirty") +) + +type Config struct { + MigrationsTable string + DatabaseName string + SchemaName string + migrationsSchemaName string + migrationsTableName string + StatementTimeout time.Duration + MigrationsTableQuoted bool + MultiStatementEnabled bool + MultiStatementMaxSize int +} + +type Postgres struct { + // Locking and unlocking need to use the same connection + conn *sql.Conn + db *sql.DB + isLocked atomic.Bool + + // Open and WithInstance need to guarantee that config is never nil + config *Config +} + +func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { + if config == nil { + return nil, ErrNilConfig + } + + if err := instance.Ping(); err != nil { + return nil, err + } + + if config.DatabaseName == "" { + query := `SELECT CURRENT_DATABASE()` + var databaseName string + if err := instance.QueryRow(query).Scan(&databaseName); err != nil { + return nil, &database.Error{OrigErr: err, Query: []byte(query)} + } + + if len(databaseName) == 0 { + return nil, ErrNoDatabaseName + } + + config.DatabaseName = databaseName + } + + if config.SchemaName == "" { + query := `SELECT CURRENT_SCHEMA()` + var schemaName string + if err := instance.QueryRow(query).Scan(&schemaName); err != nil { + return nil, &database.Error{OrigErr: err, Query: []byte(query)} + } + + if len(schemaName) == 0 { + return nil, ErrNoSchema + } + + config.SchemaName = schemaName + } + + if len(config.MigrationsTable) == 0 { + config.MigrationsTable = DefaultMigrationsTable + } + + config.migrationsSchemaName = config.SchemaName + config.migrationsTableName = config.MigrationsTable + if config.MigrationsTableQuoted { + re := regexp.MustCompile(`"(.*?)"`) + result := re.FindAllStringSubmatch(config.MigrationsTable, -1) + config.migrationsTableName = result[len(result)-1][1] + if len(result) == 2 { + config.migrationsSchemaName = result[0][1] + } else if len(result) > 2 { + return nil, fmt.Errorf("\"%s\" MigrationsTable contains too many dot characters", config.MigrationsTable) + } + } + + conn, err := instance.Conn(context.Background()) + + if err != nil { + return nil, err + } + + px := &Postgres{ + conn: conn, + db: instance, + config: config, + } + + if err := px.ensureVersionTable(); err != nil { + return nil, err + } + + return px, nil +} + +func (p *Postgres) Open(url string) (database.Driver, error) { + purl, err := nurl.Parse(url) + if err != nil { + return nil, err + } + + // Driver is registered as pgx, but connection string must use postgres schema + // when making actual connection + // i.e. pgx://user:password@host:port/db => postgres://user:password@host:port/db + purl.Scheme = "postgres" + + db, err := sql.Open("pgx", migrate.FilterCustomQuery(purl).String()) + if err != nil { + return nil, err + } + + migrationsTable := purl.Query().Get("x-migrations-table") + migrationsTableQuoted := false + if s := purl.Query().Get("x-migrations-table-quoted"); len(s) > 0 { + migrationsTableQuoted, err = strconv.ParseBool(s) + if err != nil { + return nil, fmt.Errorf("Unable to parse option x-migrations-table-quoted: %w", err) + } + } + if (len(migrationsTable) > 0) && (migrationsTableQuoted) && ((migrationsTable[0] != '"') || (migrationsTable[len(migrationsTable)-1] != '"')) { + return nil, fmt.Errorf("x-migrations-table must be quoted (for instance '\"migrate\".\"schema_migrations\"') when x-migrations-table-quoted is enabled, current value is: %s", migrationsTable) + } + + statementTimeoutString := purl.Query().Get("x-statement-timeout") + statementTimeout := 0 + if statementTimeoutString != "" { + statementTimeout, err = strconv.Atoi(statementTimeoutString) + if err != nil { + return nil, err + } + } + + multiStatementMaxSize := DefaultMultiStatementMaxSize + if s := purl.Query().Get("x-multi-statement-max-size"); len(s) > 0 { + multiStatementMaxSize, err = strconv.Atoi(s) + if err != nil { + return nil, err + } + if multiStatementMaxSize <= 0 { + multiStatementMaxSize = DefaultMultiStatementMaxSize + } + } + + multiStatementEnabled := false + if s := purl.Query().Get("x-multi-statement"); len(s) > 0 { + multiStatementEnabled, err = strconv.ParseBool(s) + if err != nil { + return nil, fmt.Errorf("Unable to parse option x-multi-statement: %w", err) + } + } + + px, err := WithInstance(db, &Config{ + DatabaseName: purl.Path, + MigrationsTable: migrationsTable, + MigrationsTableQuoted: migrationsTableQuoted, + StatementTimeout: time.Duration(statementTimeout) * time.Millisecond, + MultiStatementEnabled: multiStatementEnabled, + MultiStatementMaxSize: multiStatementMaxSize, + }) + + if err != nil { + return nil, err + } + + return px, nil +} + +func (p *Postgres) Close() error { + connErr := p.conn.Close() + dbErr := p.db.Close() + if connErr != nil || dbErr != nil { + return fmt.Errorf("conn: %v, db: %v", connErr, dbErr) + } + return nil +} + +// https://www.postgresql.org/docs/9.6/static/explicit-locking.html#ADVISORY-LOCKS +func (p *Postgres) Lock() error { + return database.CasRestoreOnErr(&p.isLocked, false, true, database.ErrLocked, func() error { + aid, err := database.GenerateAdvisoryLockId(p.config.DatabaseName, p.config.migrationsSchemaName, p.config.migrationsTableName) + if err != nil { + return err + } + + // This will wait indefinitely until the lock can be acquired. + query := `SELECT pg_advisory_lock($1)` + if _, err := p.conn.ExecContext(context.Background(), query, aid); err != nil { + return &database.Error{OrigErr: err, Err: "try lock failed", Query: []byte(query)} + } + return nil + }) +} + +func (p *Postgres) Unlock() error { + return database.CasRestoreOnErr(&p.isLocked, true, false, database.ErrNotLocked, func() error { + aid, err := database.GenerateAdvisoryLockId(p.config.DatabaseName, p.config.migrationsSchemaName, p.config.migrationsTableName) + if err != nil { + return err + } + + query := `SELECT pg_advisory_unlock($1)` + if _, err := p.conn.ExecContext(context.Background(), query, aid); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + return nil + }) +} + +func (p *Postgres) Run(migration io.Reader) error { + if p.config.MultiStatementEnabled { + var err error + if e := multistmt.Parse(migration, multiStmtDelimiter, p.config.MultiStatementMaxSize, func(m []byte) bool { + if err = p.runStatement(m); err != nil { + return false + } + return true + }); e != nil { + return e + } + return err + } + migr, err := ioutil.ReadAll(migration) + if err != nil { + return err + } + return p.runStatement(migr) +} + +func (p *Postgres) runStatement(statement []byte) error { + ctx := context.Background() + if p.config.StatementTimeout != 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, p.config.StatementTimeout) + defer cancel() + } + query := string(statement) + if strings.TrimSpace(query) == "" { + return nil + } + if _, err := p.conn.ExecContext(ctx, query); err != nil { + + if pgErr, ok := err.(*pgconn.PgError); ok { + var line uint + var col uint + var lineColOK bool + line, col, lineColOK = computeLineFromPos(query, int(pgErr.Position)) + message := fmt.Sprintf("migration failed: %s", pgErr.Message) + if lineColOK { + message = fmt.Sprintf("%s (column %d)", message, col) + } + if pgErr.Detail != "" { + message = fmt.Sprintf("%s, %s", message, pgErr.Detail) + } + return database.Error{OrigErr: err, Err: message, Query: statement, Line: line} + } + return database.Error{OrigErr: err, Err: "migration failed", Query: statement} + } + return nil +} + +func computeLineFromPos(s string, pos int) (line uint, col uint, ok bool) { + // replace crlf with lf + s = strings.Replace(s, "\r\n", "\n", -1) + // pg docs: pos uses index 1 for the first character, and positions are measured in characters not bytes + runes := []rune(s) + if pos > len(runes) { + return 0, 0, false + } + sel := runes[:pos] + line = uint(runesCount(sel, newLine) + 1) + col = uint(pos - 1 - runesLastIndex(sel, newLine)) + return line, col, true +} + +const newLine = '\n' + +func runesCount(input []rune, target rune) int { + var count int + for _, r := range input { + if r == target { + count++ + } + } + return count +} + +func runesLastIndex(input []rune, target rune) int { + for i := len(input) - 1; i >= 0; i-- { + if input[i] == target { + return i + } + } + return -1 +} + +func (p *Postgres) SetVersion(version int, dirty bool) error { + tx, err := p.conn.BeginTx(context.Background(), &sql.TxOptions{}) + if err != nil { + return &database.Error{OrigErr: err, Err: "transaction start failed"} + } + + query := `TRUNCATE ` + quoteIdentifier(p.config.migrationsSchemaName) + `.` + quoteIdentifier(p.config.migrationsTableName) + if _, err := tx.Exec(query); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + // Also re-write the schema version for nil dirty versions to prevent + // empty schema version for failed down migration on the first migration + // See: https://github.com/golang-migrate/migrate/issues/330 + if version >= 0 || (version == database.NilVersion && dirty) { + query = `INSERT INTO ` + quoteIdentifier(p.config.migrationsSchemaName) + `.` + quoteIdentifier(p.config.migrationsTableName) + ` (version, dirty) VALUES ($1, $2)` + if _, err := tx.Exec(query, version, dirty); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + + if err := tx.Commit(); err != nil { + return &database.Error{OrigErr: err, Err: "transaction commit failed"} + } + + return nil +} + +func (p *Postgres) Version() (version int, dirty bool, err error) { + query := `SELECT version, dirty FROM ` + quoteIdentifier(p.config.migrationsSchemaName) + `.` + quoteIdentifier(p.config.migrationsTableName) + ` LIMIT 1` + err = p.conn.QueryRowContext(context.Background(), query).Scan(&version, &dirty) + switch { + case err == sql.ErrNoRows: + return database.NilVersion, false, nil + + case err != nil: + if e, ok := err.(*pgconn.PgError); ok { + if e.SQLState() == pgerrcode.UndefinedTable { + return database.NilVersion, false, nil + } + } + return 0, false, &database.Error{OrigErr: err, Query: []byte(query)} + + default: + return version, dirty, nil + } +} + +func (p *Postgres) Drop() (err error) { + // select all tables in current schema + query := `SELECT table_name FROM information_schema.tables WHERE table_schema=(SELECT current_schema()) AND table_type='BASE TABLE'` + tables, err := p.conn.QueryContext(context.Background(), query) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + defer func() { + if errClose := tables.Close(); errClose != nil { + err = multierror.Append(err, errClose) + } + }() + + // delete one table after another + tableNames := make([]string, 0) + for tables.Next() { + var tableName string + if err := tables.Scan(&tableName); err != nil { + return err + } + if len(tableName) > 0 { + tableNames = append(tableNames, tableName) + } + } + if err := tables.Err(); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + if len(tableNames) > 0 { + // delete one by one ... + for _, t := range tableNames { + query = `DROP TABLE IF EXISTS ` + quoteIdentifier(t) + ` CASCADE` + if _, err := p.conn.ExecContext(context.Background(), query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + } + + return nil +} + +// ensureVersionTable checks if versions table exists and, if not, creates it. +// Note that this function locks the database, which deviates from the usual +// convention of "caller locks" in the Postgres type. +func (p *Postgres) ensureVersionTable() (err error) { + if err = p.Lock(); err != nil { + return err + } + + defer func() { + if e := p.Unlock(); e != nil { + if err == nil { + err = e + } else { + err = multierror.Append(err, e) + } + } + }() + + // This block checks whether the `MigrationsTable` already exists. This is useful because it allows read only postgres + // users to also check the current version of the schema. Previously, even if `MigrationsTable` existed, the + // `CREATE TABLE IF NOT EXISTS...` query would fail because the user does not have the CREATE permission. + // Taken from https://github.com/mattes/migrate/blob/master/database/postgres/postgres.go#L258 + query := `SELECT COUNT(1) FROM information_schema.tables WHERE table_schema = $1 AND table_name = $2 LIMIT 1` + row := p.conn.QueryRowContext(context.Background(), query, p.config.migrationsSchemaName, p.config.migrationsTableName) + + var count int + err = row.Scan(&count) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + if count == 1 { + return nil + } + + query = `CREATE TABLE IF NOT EXISTS ` + quoteIdentifier(p.config.migrationsSchemaName) + `.` + quoteIdentifier(p.config.migrationsTableName) + ` (version bigint not null primary key, dirty boolean not null)` + if _, err = p.conn.ExecContext(context.Background(), query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + return nil +} + +// Copied from lib/pq implementation: https://github.com/lib/pq/blob/v1.9.0/conn.go#L1611 +func quoteIdentifier(name string) string { + end := strings.IndexRune(name, 0) + if end > -1 { + name = name[:end] + } + return `"` + strings.Replace(name, `"`, `""`, -1) + `"` +} diff --git a/database/pgx/pgx_test.go b/database/pgx/pgx_test.go new file mode 100644 index 000000000..9d55c4106 --- /dev/null +++ b/database/pgx/pgx_test.go @@ -0,0 +1,762 @@ +package pgx + +// error codes https://github.com/jackc/pgerrcode/blob/master/errcode.go + +import ( + "context" + "database/sql" + sqldriver "database/sql/driver" + "errors" + "fmt" + "log" + + "io" + "strconv" + "strings" + "sync" + "testing" + + "github.com/golang-migrate/migrate/v4" + + "github.com/dhui/dktest" + + "github.com/golang-migrate/migrate/v4/database" + dt "github.com/golang-migrate/migrate/v4/database/testing" + "github.com/golang-migrate/migrate/v4/dktesting" + _ "github.com/golang-migrate/migrate/v4/source/file" +) + +const ( + pgPassword = "postgres" +) + +var ( + opts = dktest.Options{ + Env: map[string]string{"POSTGRES_PASSWORD": pgPassword}, + PortRequired: true, ReadyFunc: isReady} + // Supported versions: https://www.postgresql.org/support/versioning/ + specs = []dktesting.ContainerSpec{ + {ImageName: "postgres:9.5", Options: opts}, + {ImageName: "postgres:9.6", Options: opts}, + {ImageName: "postgres:10", Options: opts}, + {ImageName: "postgres:11", Options: opts}, + {ImageName: "postgres:12", Options: opts}, + } +) + +func pgConnectionString(host, port string, options ...string) string { + options = append(options, "sslmode=disable") + return fmt.Sprintf("postgres://postgres:%s@%s:%s/postgres?%s", pgPassword, host, port, strings.Join(options, "&")) +} + +func isReady(ctx context.Context, c dktest.ContainerInfo) bool { + ip, port, err := c.FirstPort() + if err != nil { + return false + } + + db, err := sql.Open("pgx", pgConnectionString(ip, port)) + if err != nil { + return false + } + defer func() { + if err := db.Close(); err != nil { + log.Println("close error:", err) + } + }() + if err = db.PingContext(ctx); err != nil { + switch err { + case sqldriver.ErrBadConn, io.EOF: + return false + default: + log.Println(err) + } + return false + } + + return true +} + +func mustRun(t *testing.T, d database.Driver, statements []string) { + for _, statement := range statements { + if err := d.Run(strings.NewReader(statement)); err != nil { + t.Fatal(err) + } + } +} + +func Test(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := pgConnectionString(ip, port) + p := &Postgres{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + dt.Test(t, d, []byte("SELECT 1")) + }) +} + +func TestMigrate(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := pgConnectionString(ip, port) + p := &Postgres{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "pgx", d) + if err != nil { + t.Fatal(err) + } + dt.TestMigrate(t, m) + }) +} + +func TestMultipleStatements(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := pgConnectionString(ip, port) + p := &Postgres{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + if err := d.Run(strings.NewReader("CREATE TABLE foo (foo text); CREATE TABLE bar (bar text);")); err != nil { + t.Fatalf("expected err to be nil, got %v", err) + } + + // make sure second table exists + var exists bool + if err := d.(*Postgres).conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'bar' AND table_schema = (SELECT current_schema()))").Scan(&exists); err != nil { + t.Fatal(err) + } + if !exists { + t.Fatalf("expected table bar to exist") + } + }) +} + +func TestMultipleStatementsInMultiStatementMode(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := pgConnectionString(ip, port, "x-multi-statement=true") + p := &Postgres{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + if err := d.Run(strings.NewReader("CREATE TABLE foo (foo text); CREATE INDEX CONCURRENTLY idx_foo ON foo (foo);")); err != nil { + t.Fatalf("expected err to be nil, got %v", err) + } + + // make sure created index exists + var exists bool + if err := d.(*Postgres).conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = (SELECT current_schema()) AND indexname = 'idx_foo')").Scan(&exists); err != nil { + t.Fatal(err) + } + if !exists { + t.Fatalf("expected table bar to exist") + } + }) +} + +func TestErrorParsing(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := pgConnectionString(ip, port) + p := &Postgres{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + + wantErr := `migration failed: syntax error at or near "TABLEE" (column 37) in line 1: CREATE TABLE foo ` + + `(foo text); CREATE TABLEE bar (bar text); (details: ERROR: syntax error at or near "TABLEE" (SQLSTATE 42601))` + if err := d.Run(strings.NewReader("CREATE TABLE foo (foo text); CREATE TABLEE bar (bar text);")); err == nil { + t.Fatal("expected err but got nil") + } else if err.Error() != wantErr { + t.Fatalf("expected '%s' but got '%s'", wantErr, err.Error()) + } + }) +} + +func TestFilterCustomQuery(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := pgConnectionString(ip, port, "x-custom=foobar") + p := &Postgres{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + }) +} + +func TestWithSchema(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := pgConnectionString(ip, port) + p := &Postgres{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Fatal(err) + } + }() + + // create foobar schema + if err := d.Run(strings.NewReader("CREATE SCHEMA foobar AUTHORIZATION postgres")); err != nil { + t.Fatal(err) + } + if err := d.SetVersion(1, false); err != nil { + t.Fatal(err) + } + + // re-connect using that schema + d2, err := p.Open(pgConnectionString(ip, port, "search_path=foobar")) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d2.Close(); err != nil { + t.Fatal(err) + } + }() + + version, _, err := d2.Version() + if err != nil { + t.Fatal(err) + } + if version != database.NilVersion { + t.Fatal("expected NilVersion") + } + + // now update version and compare + if err := d2.SetVersion(2, false); err != nil { + t.Fatal(err) + } + version, _, err = d2.Version() + if err != nil { + t.Fatal(err) + } + if version != 2 { + t.Fatal("expected version 2") + } + + // meanwhile, the public schema still has the other version + version, _, err = d.Version() + if err != nil { + t.Fatal(err) + } + if version != 1 { + t.Fatal("expected version 2") + } + }) +} + +func TestMigrationTableOption(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := pgConnectionString(ip, port) + p := &Postgres{} + d, _ := p.Open(addr) + defer func() { + if err := d.Close(); err != nil { + t.Fatal(err) + } + }() + + // create migrate schema + if err := d.Run(strings.NewReader("CREATE SCHEMA migrate AUTHORIZATION postgres")); err != nil { + t.Fatal(err) + } + + // bad unquoted x-migrations-table parameter + wantErr := "x-migrations-table must be quoted (for instance '\"migrate\".\"schema_migrations\"') when x-migrations-table-quoted is enabled, current value is: migrate.schema_migrations" + d, err = p.Open(fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=migrate.schema_migrations&x-migrations-table-quoted=1", + pgPassword, ip, port)) + if (err != nil) && (err.Error() != wantErr) { + t.Fatalf("expected '%s' but got '%s'", wantErr, err.Error()) + } + + // too many quoted x-migrations-table parameters + wantErr = "\"\"migrate\".\"schema_migrations\".\"toomany\"\" MigrationsTable contains too many dot characters" + d, err = p.Open(fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=\"migrate\".\"schema_migrations\".\"toomany\"&x-migrations-table-quoted=1", + pgPassword, ip, port)) + if (err != nil) && (err.Error() != wantErr) { + t.Fatalf("expected '%s' but got '%s'", wantErr, err.Error()) + } + + // good quoted x-migrations-table parameter + d, err = p.Open(fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=\"migrate\".\"schema_migrations\"&x-migrations-table-quoted=1", + pgPassword, ip, port)) + if err != nil { + t.Fatal(err) + } + + // make sure migrate.schema_migrations table exists + var exists bool + if err := d.(*Postgres).conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'schema_migrations' AND table_schema = 'migrate')").Scan(&exists); err != nil { + t.Fatal(err) + } + if !exists { + t.Fatalf("expected table migrate.schema_migrations to exist") + } + + d, err = p.Open(fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=migrate.schema_migrations", + pgPassword, ip, port)) + if err != nil { + t.Fatal(err) + } + if err := d.(*Postgres).conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'migrate.schema_migrations' AND table_schema = (SELECT current_schema()))").Scan(&exists); err != nil { + t.Fatal(err) + } + if !exists { + t.Fatalf("expected table 'migrate.schema_migrations' to exist") + } + + }) +} + +func TestFailToCreateTableWithoutPermissions(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := pgConnectionString(ip, port) + + // Check that opening the postgres connection returns NilVersion + p := &Postgres{} + + d, err := p.Open(addr) + + if err != nil { + t.Fatal(err) + } + + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + + // create user who is not the owner. Although we're concatenating strings in an sql statement it should be fine + // since this is a test environment and we're not expecting to the pgPassword to be malicious + mustRun(t, d, []string{ + "CREATE USER not_owner WITH ENCRYPTED PASSWORD '" + pgPassword + "'", + "CREATE SCHEMA barfoo AUTHORIZATION postgres", + "GRANT USAGE ON SCHEMA barfoo TO not_owner", + "REVOKE CREATE ON SCHEMA barfoo FROM PUBLIC", + "REVOKE CREATE ON SCHEMA barfoo FROM not_owner", + }) + + // re-connect using that schema + d2, err := p.Open(fmt.Sprintf("postgres://not_owner:%s@%v:%v/postgres?sslmode=disable&search_path=barfoo", + pgPassword, ip, port)) + + defer func() { + if d2 == nil { + return + } + if err := d2.Close(); err != nil { + t.Fatal(err) + } + }() + + var e *database.Error + if !errors.As(err, &e) || err == nil { + t.Fatal("Unexpected error, want permission denied error. Got: ", err) + } + + if !strings.Contains(e.OrigErr.Error(), "permission denied for schema barfoo") { + t.Fatal(e) + } + + // re-connect using that x-migrations-table and x-migrations-table-quoted + d2, err = p.Open(fmt.Sprintf("postgres://not_owner:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=\"barfoo\".\"schema_migrations\"&x-migrations-table-quoted=1", + pgPassword, ip, port)) + + if !errors.As(err, &e) || err == nil { + t.Fatal("Unexpected error, want permission denied error. Got: ", err) + } + + if !strings.Contains(e.OrigErr.Error(), "permission denied for schema barfoo") { + t.Fatal(e) + } + }) +} + +func TestCheckBeforeCreateTable(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := pgConnectionString(ip, port) + + // Check that opening the postgres connection returns NilVersion + p := &Postgres{} + + d, err := p.Open(addr) + + if err != nil { + t.Fatal(err) + } + + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + + // create user who is not the owner. Although we're concatenating strings in an sql statement it should be fine + // since this is a test environment and we're not expecting to the pgPassword to be malicious + mustRun(t, d, []string{ + "CREATE USER not_owner WITH ENCRYPTED PASSWORD '" + pgPassword + "'", + "CREATE SCHEMA barfoo AUTHORIZATION postgres", + "GRANT USAGE ON SCHEMA barfoo TO not_owner", + "GRANT CREATE ON SCHEMA barfoo TO not_owner", + }) + + // re-connect using that schema + d2, err := p.Open(fmt.Sprintf("postgres://not_owner:%s@%v:%v/postgres?sslmode=disable&search_path=barfoo", + pgPassword, ip, port)) + + if err != nil { + t.Fatal(err) + } + + if err := d2.Close(); err != nil { + t.Fatal(err) + } + + // revoke privileges + mustRun(t, d, []string{ + "REVOKE CREATE ON SCHEMA barfoo FROM PUBLIC", + "REVOKE CREATE ON SCHEMA barfoo FROM not_owner", + }) + + // re-connect using that schema + d3, err := p.Open(fmt.Sprintf("postgres://not_owner:%s@%v:%v/postgres?sslmode=disable&search_path=barfoo", + pgPassword, ip, port)) + + if err != nil { + t.Fatal(err) + } + + version, _, err := d3.Version() + + if err != nil { + t.Fatal(err) + } + + if version != database.NilVersion { + t.Fatal("Unexpected version, want database.NilVersion. Got: ", version) + } + + defer func() { + if err := d3.Close(); err != nil { + t.Fatal(err) + } + }() + }) +} + +func TestParallelSchema(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := pgConnectionString(ip, port) + p := &Postgres{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + + // create foo and bar schemas + if err := d.Run(strings.NewReader("CREATE SCHEMA foo AUTHORIZATION postgres")); err != nil { + t.Fatal(err) + } + if err := d.Run(strings.NewReader("CREATE SCHEMA bar AUTHORIZATION postgres")); err != nil { + t.Fatal(err) + } + + // re-connect using that schemas + dfoo, err := p.Open(pgConnectionString(ip, port, "search_path=foo")) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := dfoo.Close(); err != nil { + t.Error(err) + } + }() + + dbar, err := p.Open(pgConnectionString(ip, port, "search_path=bar")) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := dbar.Close(); err != nil { + t.Error(err) + } + }() + + if err := dfoo.Lock(); err != nil { + t.Fatal(err) + } + + if err := dbar.Lock(); err != nil { + t.Fatal(err) + } + + if err := dbar.Unlock(); err != nil { + t.Fatal(err) + } + + if err := dfoo.Unlock(); err != nil { + t.Fatal(err) + } + }) +} + +func TestPostgres_Lock(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := pgConnectionString(ip, port) + p := &Postgres{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + + dt.Test(t, d, []byte("SELECT 1")) + + ps := d.(*Postgres) + + err = ps.Lock() + if err != nil { + t.Fatal(err) + } + + err = ps.Unlock() + if err != nil { + t.Fatal(err) + } + + err = ps.Lock() + if err != nil { + t.Fatal(err) + } + + err = ps.Unlock() + if err != nil { + t.Fatal(err) + } + }) +} + +func TestWithInstance_Concurrent(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + // The number of concurrent processes running WithInstance + const concurrency = 30 + + // We can instantiate a single database handle because it is + // actually a connection pool, and so, each of the below go + // routines will have a high probability of using a separate + // connection, which is something we want to exercise. + db, err := sql.Open("pgx", pgConnectionString(ip, port)) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := db.Close(); err != nil { + t.Error(err) + } + }() + + db.SetMaxIdleConns(concurrency) + db.SetMaxOpenConns(concurrency) + + var wg sync.WaitGroup + defer wg.Wait() + + wg.Add(concurrency) + for i := 0; i < concurrency; i++ { + go func(i int) { + defer wg.Done() + _, err := WithInstance(db, &Config{}) + if err != nil { + t.Errorf("process %d error: %s", i, err) + } + }(i) + } + }) +} +func Test_computeLineFromPos(t *testing.T) { + testcases := []struct { + pos int + wantLine uint + wantCol uint + input string + wantOk bool + }{ + { + 15, 2, 6, "SELECT *\nFROM foo", true, // foo table does not exists + }, + { + 16, 3, 6, "SELECT *\n\nFROM foo", true, // foo table does not exists, empty line + }, + { + 25, 3, 7, "SELECT *\nFROM foo\nWHERE x", true, // x column error + }, + { + 27, 5, 7, "SELECT *\n\nFROM foo\n\nWHERE x", true, // x column error, empty lines + }, + { + 10, 2, 1, "SELECT *\nFROMM foo", true, // FROMM typo + }, + { + 11, 3, 1, "SELECT *\n\nFROMM foo", true, // FROMM typo, empty line + }, + { + 17, 2, 8, "SELECT *\nFROM foo", true, // last character + }, + { + 18, 0, 0, "SELECT *\nFROM foo", false, // invalid position + }, + } + for i, tc := range testcases { + t.Run("tc"+strconv.Itoa(i), func(t *testing.T) { + run := func(crlf bool, nonASCII bool) { + var name string + if crlf { + name = "crlf" + } else { + name = "lf" + } + if nonASCII { + name += "-nonascii" + } else { + name += "-ascii" + } + t.Run(name, func(t *testing.T) { + input := tc.input + if crlf { + input = strings.Replace(input, "\n", "\r\n", -1) + } + if nonASCII { + input = strings.Replace(input, "FROM", "FRÖM", -1) + } + gotLine, gotCol, gotOK := computeLineFromPos(input, tc.pos) + + if tc.wantOk { + t.Logf("pos %d, want %d:%d, %#v", tc.pos, tc.wantLine, tc.wantCol, input) + } + + if gotOK != tc.wantOk { + t.Fatalf("expected ok %v but got %v", tc.wantOk, gotOK) + } + if gotLine != tc.wantLine { + t.Fatalf("expected line %d but got %d", tc.wantLine, gotLine) + } + if gotCol != tc.wantCol { + t.Fatalf("expected col %d but got %d", tc.wantCol, gotCol) + } + }) + } + run(false, false) + run(true, false) + run(false, true) + run(true, true) + }) + } +} diff --git a/database/postgres/README.md b/database/postgres/README.md index f6312392b..bc823f4e1 100644 --- a/database/postgres/README.md +++ b/database/postgres/README.md @@ -5,6 +5,10 @@ | URL Query | WithInstance Config | Description | |------------|---------------------|-------------| | `x-migrations-table` | `MigrationsTable` | Name of the migrations table | +| `x-migrations-table-quoted` | `MigrationsTableQuoted` | By default, migrate quotes the migration table for SQL injection safety reasons. This option disable quoting and naively checks that you have quoted the migration table name. e.g. `"my_schema"."schema_migrations"` | +| `x-statement-timeout` | `StatementTimeout` | Abort any statement that takes more than the specified number of milliseconds | +| `x-multi-statement` | `MultiStatementEnabled` | Enable multi-statement execution (default: false) | +| `x-multi-statement-max-size` | `MultiStatementMaxSize` | Maximum size of single statement in bytes (default: 10MB) | | `dbname` | `DatabaseName` | The name of the database to connect to | | `search_path` | | This variable specifies the order in which schemas are searched when an object is referenced by a simple name with no schema specified. | | `user` | | The user to sign in as | @@ -26,3 +30,10 @@ 2. Wrap your existing migrations in transactions ([BEGIN/COMMIT](https://www.postgresql.org/docs/current/static/transaction-iso.html)) if you use multiple statements within one migration. 3. Download and install the latest migrate version. 4. Force the current migration version with `migrate force `. + +## Multi-statement mode + +In PostgreSQL running multiple SQL statements in one `Exec` executes them inside a transaction. Sometimes this +behavior is not desirable because some statements can be only run outside of transaction (e.g. +`CREATE INDEX CONCURRENTLY`). If you want to use `CREATE INDEX CONCURRENTLY` without activating multi-statement mode +you have to put such statements in a separate migration files. diff --git a/database/postgres/TUTORIAL.md b/database/postgres/TUTORIAL.md new file mode 100644 index 000000000..0f19c56ff --- /dev/null +++ b/database/postgres/TUTORIAL.md @@ -0,0 +1,167 @@ +# PostgreSQL tutorial for beginners + +## Create/configure database + +For the purpose of this tutorial let's create PostgreSQL database called `example`. +Our user here is `postgres`, password `password`, and host is `localhost`. +``` +psql -h localhost -U postgres -w -c "create database example;" +``` +When using Migrate CLI we need to pass to database URL. Let's export it to a variable for convenience: +``` +export POSTGRESQL_URL='postgres://postgres:password@localhost:5432/example?sslmode=disable' +``` +`sslmode=disable` means that the connection with our database will not be encrypted. Enabling it is left as an exercise. + +You can find further description of database URLs [here](README.md#database-urls). + +## Create migrations +Let's create table called `users`: +``` +migrate create -ext sql -dir db/migrations -seq create_users_table +``` +If there were no errors, we should have two files available under `db/migrations` folder: +- 000001_create_users_table.down.sql +- 000001_create_users_table.up.sql + +Note the `sql` extension that we provided. + +In the `.up.sql` file let's create the table: +``` +CREATE TABLE IF NOT EXISTS users( + user_id serial PRIMARY KEY, + username VARCHAR (50) UNIQUE NOT NULL, + password VARCHAR (50) NOT NULL, + email VARCHAR (300) UNIQUE NOT NULL +); +``` +And in the `.down.sql` let's delete it: +``` +DROP TABLE IF EXISTS users; +``` +By adding `IF EXISTS/IF NOT EXISTS` we are making migrations idempotent - you can read more about idempotency in [getting started](../../GETTING_STARTED.md#create-migrations) + +## Run migrations +``` +migrate -database ${POSTGRESQL_URL} -path db/migrations up +``` +Let's check if the table was created properly by running `psql example -c "\d users"`. +The output you are supposed to see: +``` + Table "public.users" + Column | Type | Modifiers +----------+------------------------+--------------------------------------------------------- + user_id | integer | not null default nextval('users_user_id_seq'::regclass) + username | character varying(50) | not null + password | character varying(50) | not null + email | character varying(300) | not null +Indexes: + "users_pkey" PRIMARY KEY, btree (user_id) + "users_email_key" UNIQUE CONSTRAINT, btree (email) + "users_username_key" UNIQUE CONSTRAINT, btree (username) +``` +Great! Now let's check if running reverse migration also works: +``` +migrate -database ${POSTGRESQL_URL} -path db/migrations down +``` +Make sure to check if your database changed as expected in this case as well. + +## Database transactions + +To show database transactions usage, let's create another set of migrations by running: +``` +migrate create -ext sql -dir db/migrations -seq add_mood_to_users +``` +Again, it should create for us two migrations files: +- 000002_add_mood_to_users.down.sql +- 000002_add_mood_to_users.up.sql + +In Postgres, when we want our queries to be done in a transaction, we need to wrap it with `BEGIN` and `COMMIT` commands. +In our example, we are going to add a column to our database that can only accept enumerable values or NULL. +Migration up: +``` +BEGIN; + +CREATE TYPE enum_mood AS ENUM ( + 'happy', + 'sad', + 'neutral' +); +ALTER TABLE users ADD COLUMN mood enum_mood; + +COMMIT; +``` +Migration down: +``` +BEGIN; + +ALTER TABLE users DROP COLUMN mood; +DROP TYPE enum_mood; + +COMMIT; +``` + +Now we can run our new migration and check the database: +``` +migrate -database ${POSTGRESQL_URL} -path db/migrations up +psql example -c "\d users" +``` +Expected output: +``` + Table "public.users" + Column | Type | Modifiers +----------+------------------------+--------------------------------------------------------- + user_id | integer | not null default nextval('users_user_id_seq'::regclass) + username | character varying(50) | not null + password | character varying(50) | not null + email | character varying(300) | not null + mood | enum_mood | +Indexes: + "users_pkey" PRIMARY KEY, btree (user_id) + "users_email_key" UNIQUE CONSTRAINT, btree (email) + "users_username_key" UNIQUE CONSTRAINT, btree (username) +``` + +## Optional: Run migrations within your Go app +Here is a very simple app running migrations for the above configuration: +``` +import ( + "log" + + "github.com/golang-migrate/migrate/v4" + _ "github.com/golang-migrate/migrate/v4/database/postgres" + _ "github.com/golang-migrate/migrate/v4/source/file" +) + +func main() { + m, err := migrate.New( + "file://db/migrations", + "postgres://postgres:postgres@localhost:5432/example?sslmode=disable") + if err != nil { + log.Fatal(err) + } + if err := m.Up(); err != nil { + log.Fatal(err) + } +} +``` +You can find details [here](README.md#use-in-your-go-project) + +## Fix issue where migrations run twice + +When the schema and role names are the same, you might run into issues if you create this schema using migrations. +This is caused by the fact that the [default `search_path`](https://www.postgresql.org/docs/current/ddl-schemas.html#DDL-SCHEMAS-PATH) is `"$user", public`. +In the first run (with an empty database) the migrate table is created in `public`. +When the migrations create the `$user` schema, the next run will store (a new) migrate table in this schema (due to order of schemas in `search_path`) and tries to apply all migrations again (most likely failing). + +To solve this you need to change the default `search_path` by removing the `$user` component, so the migrate table is always stored in the (available) `public` schema. +This can be done using the [`search_path` query parameter in the URL](https://github.com/jexia/migrate/blob/fix-postgres-version-table/database/postgres/README.md#postgres). + +For example to force the migrations table in the public schema you can use: +``` +export POSTGRESQL_URL='postgres://postgres:password@localhost:5432/example?sslmode=disable&search_path=public' +``` + +Note that you need to explicitly add the schema names to the table names in your migrations when you to modify the tables of the non-public schema. + +Alternatively you can add the non-public schema manually (before applying the migrations) if that is possible in your case and let the tool store the migrations table in this schema as well. diff --git a/database/postgres/postgres.go b/database/postgres/postgres.go index 8f8faf95f..82919171d 100644 --- a/database/postgres/postgres.go +++ b/database/postgres/postgres.go @@ -1,3 +1,4 @@ +//go:build go1.9 // +build go1.9 package postgres @@ -6,14 +7,19 @@ import ( "context" "database/sql" "fmt" + "go.uber.org/atomic" "io" "io/ioutil" nurl "net/url" + "regexp" "strconv" "strings" + "time" "github.com/golang-migrate/migrate/v4" "github.com/golang-migrate/migrate/v4/database" + "github.com/golang-migrate/migrate/v4/database/multistmt" + multierror "github.com/hashicorp/go-multierror" "github.com/lib/pq" ) @@ -23,7 +29,12 @@ func init() { database.Register("postgresql", &db) } -var DefaultMigrationsTable = "schema_migrations" +var ( + multiStmtDelimiter = []byte(";") + + DefaultMigrationsTable = "schema_migrations" + DefaultMultiStatementMaxSize = 10 * 1 << 20 // 10 MB +) var ( ErrNilConfig = fmt.Errorf("no config") @@ -33,17 +44,24 @@ var ( ) type Config struct { - MigrationsTable string - DatabaseName string + MigrationsTable string + MigrationsTableQuoted bool + MultiStatementEnabled bool + DatabaseName string + SchemaName string + migrationsSchemaName string + migrationsTableName string + StatementTimeout time.Duration + MultiStatementMaxSize int } type Postgres struct { // Locking and unlocking need to use the same connection conn *sql.Conn db *sql.DB - isLocked bool + isLocked atomic.Bool - // Open and WithInstance need to garantuee that config is never nil + // Open and WithInstance need to guarantee that config is never nil config *Config } @@ -56,22 +74,51 @@ func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { return nil, err } - query := `SELECT CURRENT_DATABASE()` - var databaseName string - if err := instance.QueryRow(query).Scan(&databaseName); err != nil { - return nil, &database.Error{OrigErr: err, Query: []byte(query)} - } + if config.DatabaseName == "" { + query := `SELECT CURRENT_DATABASE()` + var databaseName string + if err := instance.QueryRow(query).Scan(&databaseName); err != nil { + return nil, &database.Error{OrigErr: err, Query: []byte(query)} + } - if len(databaseName) == 0 { - return nil, ErrNoDatabaseName + if len(databaseName) == 0 { + return nil, ErrNoDatabaseName + } + + config.DatabaseName = databaseName } - config.DatabaseName = databaseName + if config.SchemaName == "" { + query := `SELECT CURRENT_SCHEMA()` + var schemaName string + if err := instance.QueryRow(query).Scan(&schemaName); err != nil { + return nil, &database.Error{OrigErr: err, Query: []byte(query)} + } + + if len(schemaName) == 0 { + return nil, ErrNoSchema + } + + config.SchemaName = schemaName + } if len(config.MigrationsTable) == 0 { config.MigrationsTable = DefaultMigrationsTable } + config.migrationsSchemaName = config.SchemaName + config.migrationsTableName = config.MigrationsTable + if config.MigrationsTableQuoted { + re := regexp.MustCompile(`"(.*?)"`) + result := re.FindAllStringSubmatch(config.MigrationsTable, -1) + config.migrationsTableName = result[len(result)-1][1] + if len(result) == 2 { + config.migrationsSchemaName = result[0][1] + } else if len(result) > 2 { + return nil, fmt.Errorf("\"%s\" MigrationsTable contains too many dot characters", config.MigrationsTable) + } + } + conn, err := instance.Conn(context.Background()) if err != nil { @@ -103,14 +150,54 @@ func (p *Postgres) Open(url string) (database.Driver, error) { } migrationsTable := purl.Query().Get("x-migrations-table") - if len(migrationsTable) == 0 { - migrationsTable = DefaultMigrationsTable + migrationsTableQuoted := false + if s := purl.Query().Get("x-migrations-table-quoted"); len(s) > 0 { + migrationsTableQuoted, err = strconv.ParseBool(s) + if err != nil { + return nil, fmt.Errorf("Unable to parse option x-migrations-table-quoted: %w", err) + } + } + if (len(migrationsTable) > 0) && (migrationsTableQuoted) && ((migrationsTable[0] != '"') || (migrationsTable[len(migrationsTable)-1] != '"')) { + return nil, fmt.Errorf("x-migrations-table must be quoted (for instance '\"migrate\".\"schema_migrations\"') when x-migrations-table-quoted is enabled, current value is: %s", migrationsTable) + } + + statementTimeoutString := purl.Query().Get("x-statement-timeout") + statementTimeout := 0 + if statementTimeoutString != "" { + statementTimeout, err = strconv.Atoi(statementTimeoutString) + if err != nil { + return nil, err + } + } + + multiStatementMaxSize := DefaultMultiStatementMaxSize + if s := purl.Query().Get("x-multi-statement-max-size"); len(s) > 0 { + multiStatementMaxSize, err = strconv.Atoi(s) + if err != nil { + return nil, err + } + if multiStatementMaxSize <= 0 { + multiStatementMaxSize = DefaultMultiStatementMaxSize + } + } + + multiStatementEnabled := false + if s := purl.Query().Get("x-multi-statement"); len(s) > 0 { + multiStatementEnabled, err = strconv.ParseBool(s) + if err != nil { + return nil, fmt.Errorf("Unable to parse option x-multi-statement: %w", err) + } } px, err := WithInstance(db, &Config{ - DatabaseName: purl.Path, - MigrationsTable: migrationsTable, + DatabaseName: purl.Path, + MigrationsTable: migrationsTable, + MigrationsTableQuoted: migrationsTableQuoted, + StatementTimeout: time.Duration(statementTimeout) * time.Millisecond, + MultiStatementEnabled: multiStatementEnabled, + MultiStatementMaxSize: multiStatementMaxSize, }) + if err != nil { return nil, err } @@ -129,53 +216,69 @@ func (p *Postgres) Close() error { // https://www.postgresql.org/docs/9.6/static/explicit-locking.html#ADVISORY-LOCKS func (p *Postgres) Lock() error { - if p.isLocked { - return database.ErrLocked - } - - aid, err := database.GenerateAdvisoryLockId(p.config.DatabaseName) - if err != nil { - return err - } + return database.CasRestoreOnErr(&p.isLocked, false, true, database.ErrLocked, func() error { + aid, err := database.GenerateAdvisoryLockId(p.config.DatabaseName, p.config.migrationsSchemaName, p.config.migrationsTableName) + if err != nil { + return err + } - // This will either obtain the lock immediately and return true, - // or return false if the lock cannot be acquired immediately. - query := `SELECT pg_advisory_lock($1)` - if _, err := p.conn.ExecContext(context.Background(), query, aid); err != nil { - return &database.Error{OrigErr: err, Err: "try lock failed", Query: []byte(query)} - } + // This will wait indefinitely until the lock can be acquired. + query := `SELECT pg_advisory_lock($1)` + if _, err := p.conn.ExecContext(context.Background(), query, aid); err != nil { + return &database.Error{OrigErr: err, Err: "try lock failed", Query: []byte(query)} + } - p.isLocked = true - return nil + return nil + }) } func (p *Postgres) Unlock() error { - if !p.isLocked { - return nil - } - - aid, err := database.GenerateAdvisoryLockId(p.config.DatabaseName) - if err != nil { - return err - } + return database.CasRestoreOnErr(&p.isLocked, true, false, database.ErrNotLocked, func() error { + aid, err := database.GenerateAdvisoryLockId(p.config.DatabaseName, p.config.migrationsSchemaName, p.config.migrationsTableName) + if err != nil { + return err + } - query := `SELECT pg_advisory_unlock($1)` - if _, err := p.conn.ExecContext(context.Background(), query, aid); err != nil { - return &database.Error{OrigErr: err, Query: []byte(query)} - } - p.isLocked = false - return nil + query := `SELECT pg_advisory_unlock($1)` + if _, err := p.conn.ExecContext(context.Background(), query, aid); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + return nil + }) } func (p *Postgres) Run(migration io.Reader) error { + if p.config.MultiStatementEnabled { + var err error + if e := multistmt.Parse(migration, multiStmtDelimiter, p.config.MultiStatementMaxSize, func(m []byte) bool { + if err = p.runStatement(m); err != nil { + return false + } + return true + }); e != nil { + return e + } + return err + } migr, err := ioutil.ReadAll(migration) if err != nil { return err } + return p.runStatement(migr) +} - // run migration - query := string(migr[:]) - if _, err := p.conn.ExecContext(context.Background(), query); err != nil { +func (p *Postgres) runStatement(statement []byte) error { + ctx := context.Background() + if p.config.StatementTimeout != 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, p.config.StatementTimeout) + defer cancel() + } + query := string(statement) + if strings.TrimSpace(query) == "" { + return nil + } + if _, err := p.conn.ExecContext(ctx, query); err != nil { if pgErr, ok := err.(*pq.Error); ok { var line uint var col uint @@ -192,11 +295,10 @@ func (p *Postgres) Run(migration io.Reader) error { if pgErr.Detail != "" { message = fmt.Sprintf("%s, %s", message, pgErr.Detail) } - return database.Error{OrigErr: err, Err: message, Query: migr, Line: line} + return database.Error{OrigErr: err, Err: message, Query: statement, Line: line} } - return database.Error{OrigErr: err, Err: "migration failed", Query: migr} + return database.Error{OrigErr: err, Err: "migration failed", Query: statement} } - return nil } @@ -241,16 +343,23 @@ func (p *Postgres) SetVersion(version int, dirty bool) error { return &database.Error{OrigErr: err, Err: "transaction start failed"} } - query := `TRUNCATE "` + p.config.MigrationsTable + `"` + query := `TRUNCATE ` + pq.QuoteIdentifier(p.config.migrationsSchemaName) + `.` + pq.QuoteIdentifier(p.config.migrationsTableName) if _, err := tx.Exec(query); err != nil { - tx.Rollback() + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } return &database.Error{OrigErr: err, Query: []byte(query)} } - if version >= 0 { - query = `INSERT INTO "` + p.config.MigrationsTable + `" (version, dirty) VALUES ($1, $2)` + // Also re-write the schema version for nil dirty versions to prevent + // empty schema version for failed down migration on the first migration + // See: https://github.com/golang-migrate/migrate/issues/330 + if version >= 0 || (version == database.NilVersion && dirty) { + query = `INSERT INTO ` + pq.QuoteIdentifier(p.config.migrationsSchemaName) + `.` + pq.QuoteIdentifier(p.config.migrationsTableName) + ` (version, dirty) VALUES ($1, $2)` if _, err := tx.Exec(query, version, dirty); err != nil { - tx.Rollback() + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } return &database.Error{OrigErr: err, Query: []byte(query)} } } @@ -263,7 +372,7 @@ func (p *Postgres) SetVersion(version int, dirty bool) error { } func (p *Postgres) Version() (version int, dirty bool, err error) { - query := `SELECT version, dirty FROM "` + p.config.MigrationsTable + `" LIMIT 1` + query := `SELECT version, dirty FROM ` + pq.QuoteIdentifier(p.config.migrationsSchemaName) + `.` + pq.QuoteIdentifier(p.config.migrationsTableName) + ` LIMIT 1` err = p.conn.QueryRowContext(context.Background(), query).Scan(&version, &dirty) switch { case err == sql.ErrNoRows: @@ -282,14 +391,18 @@ func (p *Postgres) Version() (version int, dirty bool, err error) { } } -func (p *Postgres) Drop() error { +func (p *Postgres) Drop() (err error) { // select all tables in current schema query := `SELECT table_name FROM information_schema.tables WHERE table_schema=(SELECT current_schema()) AND table_type='BASE TABLE'` tables, err := p.conn.QueryContext(context.Background(), query) if err != nil { return &database.Error{OrigErr: err, Query: []byte(query)} } - defer tables.Close() + defer func() { + if errClose := tables.Close(); errClose != nil { + err = multierror.Append(err, errClose) + } + }() // delete one table after another tableNames := make([]string, 0) @@ -302,38 +415,62 @@ func (p *Postgres) Drop() error { tableNames = append(tableNames, tableName) } } + if err := tables.Err(); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } if len(tableNames) > 0 { // delete one by one ... for _, t := range tableNames { - query = `DROP TABLE IF EXISTS ` + t + ` CASCADE` + query = `DROP TABLE IF EXISTS ` + pq.QuoteIdentifier(t) + ` CASCADE` if _, err := p.conn.ExecContext(context.Background(), query); err != nil { return &database.Error{OrigErr: err, Query: []byte(query)} } } - if err := p.ensureVersionTable(); err != nil { - return err - } } return nil } -func (p *Postgres) ensureVersionTable() error { - // check if migration table exists +// ensureVersionTable checks if versions table exists and, if not, creates it. +// Note that this function locks the database, which deviates from the usual +// convention of "caller locks" in the Postgres type. +func (p *Postgres) ensureVersionTable() (err error) { + if err = p.Lock(); err != nil { + return err + } + + defer func() { + if e := p.Unlock(); e != nil { + if err == nil { + err = e + } else { + err = multierror.Append(err, e) + } + } + }() + + // This block checks whether the `MigrationsTable` already exists. This is useful because it allows read only postgres + // users to also check the current version of the schema. Previously, even if `MigrationsTable` existed, the + // `CREATE TABLE IF NOT EXISTS...` query would fail because the user does not have the CREATE permission. + // Taken from https://github.com/mattes/migrate/blob/master/database/postgres/postgres.go#L258 + query := `SELECT COUNT(1) FROM information_schema.tables WHERE table_schema = $1 AND table_name = $2 LIMIT 1` + row := p.conn.QueryRowContext(context.Background(), query, p.config.migrationsSchemaName, p.config.migrationsTableName) + var count int - query := `SELECT COUNT(1) FROM information_schema.tables WHERE table_name = $1 AND table_schema = (SELECT current_schema()) LIMIT 1` - if err := p.conn.QueryRowContext(context.Background(), query, p.config.MigrationsTable).Scan(&count); err != nil { + err = row.Scan(&count) + if err != nil { return &database.Error{OrigErr: err, Query: []byte(query)} } + if count == 1 { return nil } - // if not, create the empty migration table - query = `CREATE TABLE "` + p.config.MigrationsTable + `" (version bigint not null primary key, dirty boolean not null)` - if _, err := p.conn.ExecContext(context.Background(), query); err != nil { + query = `CREATE TABLE IF NOT EXISTS ` + pq.QuoteIdentifier(p.config.migrationsSchemaName) + `.` + pq.QuoteIdentifier(p.config.migrationsTableName) + ` (version bigint not null primary key, dirty boolean not null)` + if _, err = p.conn.ExecContext(context.Background(), query); err != nil { return &database.Error{OrigErr: err, Query: []byte(query)} } + return nil } diff --git a/database/postgres/postgres_test.go b/database/postgres/postgres_test.go index 9da9938ed..c4ed3560f 100644 --- a/database/postgres/postgres_test.go +++ b/database/postgres/postgres_test.go @@ -3,44 +3,72 @@ package postgres // error codes https://github.com/lib/pq/blob/master/error.go import ( - "bytes" "context" "database/sql" sqldriver "database/sql/driver" + "errors" "fmt" "io" + "log" "strconv" "strings" + "sync" "testing" + "github.com/golang-migrate/migrate/v4" + + "github.com/dhui/dktest" + + "github.com/golang-migrate/migrate/v4/database" dt "github.com/golang-migrate/migrate/v4/database/testing" - mt "github.com/golang-migrate/migrate/v4/testing" + "github.com/golang-migrate/migrate/v4/dktesting" + _ "github.com/golang-migrate/migrate/v4/source/file" ) -var versions = []mt.Version{ - {Image: "postgres:10"}, - {Image: "postgres:9.6"}, - {Image: "postgres:9.5"}, - {Image: "postgres:9.4"}, - {Image: "postgres:9.3"}, -} +const ( + pgPassword = "postgres" +) + +var ( + opts = dktest.Options{ + Env: map[string]string{"POSTGRES_PASSWORD": pgPassword}, + PortRequired: true, ReadyFunc: isReady} + // Supported versions: https://www.postgresql.org/support/versioning/ + specs = []dktesting.ContainerSpec{ + {ImageName: "postgres:9.5", Options: opts}, + {ImageName: "postgres:9.6", Options: opts}, + {ImageName: "postgres:10", Options: opts}, + {ImageName: "postgres:11", Options: opts}, + {ImageName: "postgres:12", Options: opts}, + } +) -func pgConnectionString(host string, port uint) string { - return fmt.Sprintf("postgres://postgres@%s:%v/postgres?sslmode=disable", host, port) +func pgConnectionString(host, port string, options ...string) string { + options = append(options, "sslmode=disable") + return fmt.Sprintf("postgres://postgres:%s@%s:%s/postgres?%s", pgPassword, host, port, strings.Join(options, "&")) } -func isReady(i mt.Instance) bool { - db, err := sql.Open("postgres", pgConnectionString(i.Host(), i.Port())) +func isReady(ctx context.Context, c dktest.ContainerInfo) bool { + ip, port, err := c.FirstPort() + if err != nil { + return false + } + + db, err := sql.Open("postgres", pgConnectionString(ip, port)) if err != nil { return false } - defer db.Close() - if err = db.Ping(); err != nil { + defer func() { + if err := db.Close(); err != nil { + log.Println("close error:", err) + } + }() + if err = db.PingContext(ctx); err != nil { switch err { case sqldriver.ErrBadConn, io.EOF: return false default: - fmt.Println(err) + log.Println(err) } return false } @@ -48,176 +76,614 @@ func isReady(i mt.Instance) bool { return true } +func mustRun(t *testing.T, d database.Driver, statements []string) { + for _, statement := range statements { + if err := d.Run(strings.NewReader(statement)); err != nil { + t.Fatal(err) + } + } +} + func Test(t *testing.T) { - mt.ParallelTest(t, versions, isReady, - func(t *testing.T, i mt.Instance) { - p := &Postgres{} - addr := pgConnectionString(i.Host(), i.Port()) - d, err := p.Open(addr) - if err != nil { - t.Fatalf("%v", err) + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := pgConnectionString(ip, port) + p := &Postgres{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) } - defer d.Close() - dt.Test(t, d, []byte("SELECT 1")) - }) + }() + dt.Test(t, d, []byte("SELECT 1")) + }) } -func TestMultiStatement(t *testing.T) { - mt.ParallelTest(t, versions, isReady, - func(t *testing.T, i mt.Instance) { - p := &Postgres{} - addr := pgConnectionString(i.Host(), i.Port()) - d, err := p.Open(addr) - if err != nil { - t.Fatalf("%v", err) - } - defer d.Close() - if err := d.Run(bytes.NewReader([]byte("CREATE TABLE foo (foo text); CREATE TABLE bar (bar text);"))); err != nil { - t.Fatalf("expected err to be nil, got %v", err) +func TestMigrate(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := pgConnectionString(ip, port) + p := &Postgres{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) } + }() + m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "postgres", d) + if err != nil { + t.Fatal(err) + } + dt.TestMigrate(t, m) + }) +} - // make sure second table exists - var exists bool - if err := d.(*Postgres).conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'bar' AND table_schema = (SELECT current_schema()))").Scan(&exists); err != nil { - t.Fatal(err) +func TestMultipleStatements(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := pgConnectionString(ip, port) + p := &Postgres{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) } - if !exists { - t.Fatalf("expected table bar to exist") + }() + if err := d.Run(strings.NewReader("CREATE TABLE foo (foo text); CREATE TABLE bar (bar text);")); err != nil { + t.Fatalf("expected err to be nil, got %v", err) + } + + // make sure second table exists + var exists bool + if err := d.(*Postgres).conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'bar' AND table_schema = (SELECT current_schema()))").Scan(&exists); err != nil { + t.Fatal(err) + } + if !exists { + t.Fatalf("expected table bar to exist") + } + }) +} + +func TestMultipleStatementsInMultiStatementMode(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := pgConnectionString(ip, port, "x-multi-statement=true") + p := &Postgres{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) } - }) + }() + if err := d.Run(strings.NewReader("CREATE TABLE foo (foo text); CREATE INDEX CONCURRENTLY idx_foo ON foo (foo);")); err != nil { + t.Fatalf("expected err to be nil, got %v", err) + } + + // make sure created index exists + var exists bool + if err := d.(*Postgres).conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = (SELECT current_schema()) AND indexname = 'idx_foo')").Scan(&exists); err != nil { + t.Fatal(err) + } + if !exists { + t.Fatalf("expected table bar to exist") + } + }) } func TestErrorParsing(t *testing.T) { - mt.ParallelTest(t, versions, isReady, - func(t *testing.T, i mt.Instance) { - p := &Postgres{} - addr := pgConnectionString(i.Host(), i.Port()) - d, err := p.Open(addr) - if err != nil { - t.Fatalf("%v", err) - } - defer d.Close() - - wantErr := `migration failed: syntax error at or near "TABLEE" (column 37) in line 1: CREATE TABLE foo ` + - `(foo text); CREATE TABLEE bar (bar text); (details: pq: syntax error at or near "TABLEE")` - if err := d.Run(bytes.NewReader([]byte("CREATE TABLE foo (foo text); CREATE TABLEE bar (bar text);"))); err == nil { - t.Fatal("expected err but got nil") - } else if err.Error() != wantErr { - t.Fatalf("expected '%s' but got '%s'", wantErr, err.Error()) + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := pgConnectionString(ip, port) + p := &Postgres{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) } - }) + }() + + wantErr := `migration failed: syntax error at or near "TABLEE" (column 37) in line 1: CREATE TABLE foo ` + + `(foo text); CREATE TABLEE bar (bar text); (details: pq: syntax error at or near "TABLEE")` + if err := d.Run(strings.NewReader("CREATE TABLE foo (foo text); CREATE TABLEE bar (bar text);")); err == nil { + t.Fatal("expected err but got nil") + } else if err.Error() != wantErr { + t.Fatalf("expected '%s' but got '%s'", wantErr, err.Error()) + } + }) } func TestFilterCustomQuery(t *testing.T) { - mt.ParallelTest(t, versions, isReady, - func(t *testing.T, i mt.Instance) { - p := &Postgres{} - addr := fmt.Sprintf("postgres://postgres@%v:%v/postgres?sslmode=disable&x-custom=foobar", i.Host(), i.Port()) - d, err := p.Open(addr) - if err != nil { - t.Fatalf("%v", err) + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&x-custom=foobar", + pgPassword, ip, port) + p := &Postgres{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) } - defer d.Close() - }) + }() + }) } func TestWithSchema(t *testing.T) { - mt.ParallelTest(t, versions, isReady, - func(t *testing.T, i mt.Instance) { - p := &Postgres{} - addr := pgConnectionString(i.Host(), i.Port()) - d, err := p.Open(addr) - if err != nil { - t.Fatalf("%v", err) - } - defer d.Close() + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } - // create foobar schema - if err := d.Run(bytes.NewReader([]byte("CREATE SCHEMA foobar AUTHORIZATION postgres"))); err != nil { + addr := pgConnectionString(ip, port) + p := &Postgres{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { t.Fatal(err) } - if err := d.SetVersion(1, false); err != nil { + }() + + // create foobar schema + if err := d.Run(strings.NewReader("CREATE SCHEMA foobar AUTHORIZATION postgres")); err != nil { + t.Fatal(err) + } + if err := d.SetVersion(1, false); err != nil { + t.Fatal(err) + } + + // re-connect using that schema + d2, err := p.Open(fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&search_path=foobar", + pgPassword, ip, port)) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d2.Close(); err != nil { t.Fatal(err) } + }() - // re-connect using that schema - d2, err := p.Open(fmt.Sprintf("postgres://postgres@%v:%v/postgres?sslmode=disable&search_path=foobar", i.Host(), i.Port())) - if err != nil { - t.Fatalf("%v", err) - } - defer d2.Close() + version, _, err := d2.Version() + if err != nil { + t.Fatal(err) + } + if version != database.NilVersion { + t.Fatal("expected NilVersion") + } + + // now update version and compare + if err := d2.SetVersion(2, false); err != nil { + t.Fatal(err) + } + version, _, err = d2.Version() + if err != nil { + t.Fatal(err) + } + if version != 2 { + t.Fatal("expected version 2") + } + + // meanwhile, the public schema still has the other version + version, _, err = d.Version() + if err != nil { + t.Fatal(err) + } + if version != 1 { + t.Fatal("expected version 2") + } + }) +} - version, _, err := d2.Version() - if err != nil { +func TestMigrationTableOption(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := pgConnectionString(ip, port) + p := &Postgres{} + d, _ := p.Open(addr) + defer func() { + if err := d.Close(); err != nil { t.Fatal(err) } - if version != -1 { - t.Fatal("expected NilVersion") + }() + + // create migrate schema + if err := d.Run(strings.NewReader("CREATE SCHEMA migrate AUTHORIZATION postgres")); err != nil { + t.Fatal(err) + } + + // bad unquoted x-migrations-table parameter + wantErr := "x-migrations-table must be quoted (for instance '\"migrate\".\"schema_migrations\"') when x-migrations-table-quoted is enabled, current value is: migrate.schema_migrations" + d, err = p.Open(fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=migrate.schema_migrations&x-migrations-table-quoted=1", + pgPassword, ip, port)) + if (err != nil) && (err.Error() != wantErr) { + t.Fatalf("expected '%s' but got '%s'", wantErr, err.Error()) + } + + // too many quoted x-migrations-table parameters + wantErr = "\"\"migrate\".\"schema_migrations\".\"toomany\"\" MigrationsTable contains too many dot characters" + d, err = p.Open(fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=\"migrate\".\"schema_migrations\".\"toomany\"&x-migrations-table-quoted=1", + pgPassword, ip, port)) + if (err != nil) && (err.Error() != wantErr) { + t.Fatalf("expected '%s' but got '%s'", wantErr, err.Error()) + } + + // good quoted x-migrations-table parameter + d, err = p.Open(fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=\"migrate\".\"schema_migrations\"&x-migrations-table-quoted=1", + pgPassword, ip, port)) + if err != nil { + t.Fatal(err) + } + + // make sure migrate.schema_migrations table exists + var exists bool + if err := d.(*Postgres).conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'schema_migrations' AND table_schema = 'migrate')").Scan(&exists); err != nil { + t.Fatal(err) + } + if !exists { + t.Fatalf("expected table migrate.schema_migrations to exist") + } + + d, err = p.Open(fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=migrate.schema_migrations", + pgPassword, ip, port)) + if err != nil { + t.Fatal(err) + } + if err := d.(*Postgres).conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'migrate.schema_migrations' AND table_schema = (SELECT current_schema()))").Scan(&exists); err != nil { + t.Fatal(err) + } + if !exists { + t.Fatalf("expected table 'migrate.schema_migrations' to exist") + } + + }) +} + +func TestFailToCreateTableWithoutPermissions(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := pgConnectionString(ip, port) + + // Check that opening the postgres connection returns NilVersion + p := &Postgres{} + + d, err := p.Open(addr) + + if err != nil { + t.Fatal(err) + } + + defer func() { + if err := d.Close(); err != nil { + t.Error(err) } + }() + + // create user who is not the owner. Although we're concatenating strings in an sql statement it should be fine + // since this is a test environment and we're not expecting to the pgPassword to be malicious + mustRun(t, d, []string{ + "CREATE USER not_owner WITH ENCRYPTED PASSWORD '" + pgPassword + "'", + "CREATE SCHEMA barfoo AUTHORIZATION postgres", + "GRANT USAGE ON SCHEMA barfoo TO not_owner", + "REVOKE CREATE ON SCHEMA barfoo FROM PUBLIC", + "REVOKE CREATE ON SCHEMA barfoo FROM not_owner", + }) - // now update version and compare - if err := d2.SetVersion(2, false); err != nil { - t.Fatal(err) + // re-connect using that schema + d2, err := p.Open(fmt.Sprintf("postgres://not_owner:%s@%v:%v/postgres?sslmode=disable&search_path=barfoo", + pgPassword, ip, port)) + + defer func() { + if d2 == nil { + return } - version, _, err = d2.Version() - if err != nil { + if err := d2.Close(); err != nil { t.Fatal(err) } - if version != 2 { - t.Fatal("expected version 2") - } + }() - // meanwhile, the public schema still has the other version - version, _, err = d.Version() - if err != nil { - t.Fatal(err) - } - if version != 1 { - t.Fatal("expected version 2") - } - }) -} + var e *database.Error + if !errors.As(err, &e) || err == nil { + t.Fatal("Unexpected error, want permission denied error. Got: ", err) + } + + if !strings.Contains(e.OrigErr.Error(), "permission denied for schema barfoo") { + t.Fatal(e) + } + + // re-connect using that x-migrations-table and x-migrations-table-quoted + d2, err = p.Open(fmt.Sprintf("postgres://not_owner:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=\"barfoo\".\"schema_migrations\"&x-migrations-table-quoted=1", + pgPassword, ip, port)) -func TestWithInstance(t *testing.T) { + if !errors.As(err, &e) || err == nil { + t.Fatal("Unexpected error, want permission denied error. Got: ", err) + } + if !strings.Contains(e.OrigErr.Error(), "permission denied for schema barfoo") { + t.Fatal(e) + } + }) } -func TestPostgres_Lock(t *testing.T) { - mt.ParallelTest(t, versions, isReady, - func(t *testing.T, i mt.Instance) { - p := &Postgres{} - addr := pgConnectionString(i.Host(), i.Port()) - d, err := p.Open(addr) - if err != nil { - t.Fatalf("%v", err) +func TestCheckBeforeCreateTable(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := pgConnectionString(ip, port) + + // Check that opening the postgres connection returns NilVersion + p := &Postgres{} + + d, err := p.Open(addr) + + if err != nil { + t.Fatal(err) + } + + defer func() { + if err := d.Close(); err != nil { + t.Error(err) } + }() + + // create user who is not the owner. Although we're concatenating strings in an sql statement it should be fine + // since this is a test environment and we're not expecting to the pgPassword to be malicious + mustRun(t, d, []string{ + "CREATE USER not_owner WITH ENCRYPTED PASSWORD '" + pgPassword + "'", + "CREATE SCHEMA barfoo AUTHORIZATION postgres", + "GRANT USAGE ON SCHEMA barfoo TO not_owner", + "GRANT CREATE ON SCHEMA barfoo TO not_owner", + }) + + // re-connect using that schema + d2, err := p.Open(fmt.Sprintf("postgres://not_owner:%s@%v:%v/postgres?sslmode=disable&search_path=barfoo", + pgPassword, ip, port)) + + if err != nil { + t.Fatal(err) + } - dt.Test(t, d, []byte("SELECT 1")) + if err := d2.Close(); err != nil { + t.Fatal(err) + } - ps := d.(*Postgres) + // revoke privileges + mustRun(t, d, []string{ + "REVOKE CREATE ON SCHEMA barfoo FROM PUBLIC", + "REVOKE CREATE ON SCHEMA barfoo FROM not_owner", + }) - err = ps.Lock() - if err != nil { + // re-connect using that schema + d3, err := p.Open(fmt.Sprintf("postgres://not_owner:%s@%v:%v/postgres?sslmode=disable&search_path=barfoo", + pgPassword, ip, port)) + + if err != nil { + t.Fatal(err) + } + + version, _, err := d3.Version() + + if err != nil { + t.Fatal(err) + } + + if version != database.NilVersion { + t.Fatal("Unexpected version, want database.NilVersion. Got: ", version) + } + + defer func() { + if err := d3.Close(); err != nil { t.Fatal(err) } + }() + }) +} - err = ps.Unlock() - if err != nil { - t.Fatal(err) +func TestParallelSchema(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := pgConnectionString(ip, port) + p := &Postgres{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) } + }() - err = ps.Lock() - if err != nil { - t.Fatal(err) + // create foo and bar schemas + if err := d.Run(strings.NewReader("CREATE SCHEMA foo AUTHORIZATION postgres")); err != nil { + t.Fatal(err) + } + if err := d.Run(strings.NewReader("CREATE SCHEMA bar AUTHORIZATION postgres")); err != nil { + t.Fatal(err) + } + + // re-connect using that schemas + dfoo, err := p.Open(fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&search_path=foo", + pgPassword, ip, port)) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := dfoo.Close(); err != nil { + t.Error(err) } + }() - err = ps.Unlock() - if err != nil { - t.Fatal(err) + dbar, err := p.Open(fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&search_path=bar", + pgPassword, ip, port)) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := dbar.Close(); err != nil { + t.Error(err) } - }) + }() + + if err := dfoo.Lock(); err != nil { + t.Fatal(err) + } + + if err := dbar.Lock(); err != nil { + t.Fatal(err) + } + + if err := dbar.Unlock(); err != nil { + t.Fatal(err) + } + + if err := dfoo.Unlock(); err != nil { + t.Fatal(err) + } + }) +} + +func TestPostgres_Lock(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := pgConnectionString(ip, port) + p := &Postgres{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + + dt.Test(t, d, []byte("SELECT 1")) + + ps := d.(*Postgres) + + err = ps.Lock() + if err != nil { + t.Fatal(err) + } + + err = ps.Unlock() + if err != nil { + t.Fatal(err) + } + + err = ps.Lock() + if err != nil { + t.Fatal(err) + } + + err = ps.Unlock() + if err != nil { + t.Fatal(err) + } + }) } +func TestWithInstance_Concurrent(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + // The number of concurrent processes running WithInstance + const concurrency = 30 + + // We can instantiate a single database handle because it is + // actually a connection pool, and so, each of the below go + // routines will have a high probability of using a separate + // connection, which is something we want to exercise. + db, err := sql.Open("postgres", pgConnectionString(ip, port)) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := db.Close(); err != nil { + t.Error(err) + } + }() + + db.SetMaxIdleConns(concurrency) + db.SetMaxOpenConns(concurrency) + + var wg sync.WaitGroup + defer wg.Wait() + + wg.Add(concurrency) + for i := 0; i < concurrency; i++ { + go func(i int) { + defer wg.Done() + _, err := WithInstance(db, &Config{}) + if err != nil { + t.Errorf("process %d error: %s", i, err) + } + }(i) + } + }) +} func Test_computeLineFromPos(t *testing.T) { testcases := []struct { pos int @@ -296,5 +762,4 @@ func Test_computeLineFromPos(t *testing.T) { run(true, true) }) } - } diff --git a/database/ql/migration/33_create_table.down.sql b/database/ql/examples/migrations/33_create_table.down.sql similarity index 100% rename from database/ql/migration/33_create_table.down.sql rename to database/ql/examples/migrations/33_create_table.down.sql diff --git a/database/ql/migration/33_create_table.up.sql b/database/ql/examples/migrations/33_create_table.up.sql similarity index 100% rename from database/ql/migration/33_create_table.up.sql rename to database/ql/examples/migrations/33_create_table.up.sql diff --git a/database/ql/migration/44_alter_table.down.sql b/database/ql/examples/migrations/44_alter_table.down.sql similarity index 100% rename from database/ql/migration/44_alter_table.down.sql rename to database/ql/examples/migrations/44_alter_table.down.sql diff --git a/database/ql/migration/44_alter_table.up.sql b/database/ql/examples/migrations/44_alter_table.up.sql similarity index 100% rename from database/ql/migration/44_alter_table.up.sql rename to database/ql/examples/migrations/44_alter_table.up.sql diff --git a/database/ql/ql.go b/database/ql/ql.go index 86b2364dd..1c4c49be6 100644 --- a/database/ql/ql.go +++ b/database/ql/ql.go @@ -3,15 +3,17 @@ package ql import ( "database/sql" "fmt" + "github.com/hashicorp/go-multierror" + "go.uber.org/atomic" "io" "io/ioutil" "strings" nurl "net/url" - _ "github.com/cznic/ql/driver" "github.com/golang-migrate/migrate/v4" "github.com/golang-migrate/migrate/v4/database" + _ "modernc.org/ql/driver" ) func init() { @@ -33,7 +35,7 @@ type Config struct { type Ql struct { db *sql.DB - isLocked bool + isLocked atomic.Bool config *Config } @@ -46,6 +48,7 @@ func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { if err := instance.Ping(); err != nil { return nil, err } + if len(config.MigrationsTable) == 0 { config.MigrationsTable = DefaultMigrationsTable } @@ -59,13 +62,31 @@ func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { } return mx, nil } -func (m *Ql) ensureVersionTable() error { + +// ensureVersionTable checks if versions table exists and, if not, creates it. +// Note that this function locks the database, which deviates from the usual +// convention of "caller locks" in the Ql type. +func (m *Ql) ensureVersionTable() (err error) { + if err = m.Lock(); err != nil { + return err + } + + defer func() { + if e := m.Unlock(); e != nil { + if err == nil { + err = e + } else { + err = multierror.Append(err, e) + } + } + }() + tx, err := m.db.Begin() if err != nil { return err } if _, err := tx.Exec(fmt.Sprintf(` - CREATE TABLE IF NOT EXISTS %s (version uint64,dirty bool); + CREATE TABLE IF NOT EXISTS %s (version uint64, dirty bool); CREATE UNIQUE INDEX IF NOT EXISTS version_unique ON %s (version); `, m.config.MigrationsTable, m.config.MigrationsTable)); err != nil { if err := tx.Rollback(); err != nil { @@ -105,13 +126,18 @@ func (m *Ql) Open(url string) (database.Driver, error) { func (m *Ql) Close() error { return m.db.Close() } -func (m *Ql) Drop() error { +func (m *Ql) Drop() (err error) { query := `SELECT Name FROM __Table` tables, err := m.db.Query(query) if err != nil { return &database.Error{OrigErr: err, Query: []byte(query)} } - defer tables.Close() + defer func() { + if errClose := tables.Close(); errClose != nil { + err = multierror.Append(err, errClose) + } + }() + tableNames := make([]string, 0) for tables.Next() { var tableName string @@ -119,11 +145,15 @@ func (m *Ql) Drop() error { return err } if len(tableName) > 0 { - if strings.HasPrefix(tableName, "__") == false { + if !strings.HasPrefix(tableName, "__") { tableNames = append(tableNames, tableName) } } } + if err := tables.Err(); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + if len(tableNames) > 0 { for _, t := range tableNames { query := "DROP TABLE " + t @@ -132,25 +162,20 @@ func (m *Ql) Drop() error { return &database.Error{OrigErr: err, Query: []byte(query)} } } - if err := m.ensureVersionTable(); err != nil { - return err - } } return nil } func (m *Ql) Lock() error { - if m.isLocked { + if !m.isLocked.CAS(false, true) { return database.ErrLocked } - m.isLocked = true return nil } func (m *Ql) Unlock() error { - if !m.isLocked { - return nil + if !m.isLocked.CAS(true, false) { + return database.ErrNotLocked } - m.isLocked = false return nil } func (m *Ql) Run(migration io.Reader) error { @@ -168,7 +193,9 @@ func (m *Ql) executeQuery(query string) error { return &database.Error{OrigErr: err, Err: "transaction start failed"} } if _, err := tx.Exec(query); err != nil { - tx.Rollback() + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } return &database.Error{OrigErr: err, Query: []byte(query)} } if err := tx.Commit(); err != nil { @@ -187,10 +214,16 @@ func (m *Ql) SetVersion(version int, dirty bool) error { return &database.Error{OrigErr: err, Query: []byte(query)} } - if version >= 0 { - query := fmt.Sprintf(`INSERT INTO %s (version, dirty) VALUES (%d, %t)`, m.config.MigrationsTable, version, dirty) - if _, err := tx.Exec(query); err != nil { - tx.Rollback() + // Also re-write the schema version for nil dirty versions to prevent + // empty schema version for failed down migration on the first migration + // See: https://github.com/golang-migrate/migrate/issues/330 + if version >= 0 || (version == database.NilVersion && dirty) { + query := fmt.Sprintf(`INSERT INTO %s (version, dirty) VALUES (uint64(?1), ?2)`, + m.config.MigrationsTable) + if _, err := tx.Exec(query, version, dirty); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } return &database.Error{OrigErr: err, Query: []byte(query)} } } diff --git a/database/ql/ql_test.go b/database/ql/ql_test.go index 5a05e355e..b9073166d 100644 --- a/database/ql/ql_test.go +++ b/database/ql/ql_test.go @@ -8,10 +8,10 @@ import ( "path/filepath" "testing" - _ "github.com/cznic/ql/driver" "github.com/golang-migrate/migrate/v4" dt "github.com/golang-migrate/migrate/v4/database/testing" _ "github.com/golang-migrate/migrate/v4/source/file" + _ "modernc.org/ql/driver" ) func Test(t *testing.T) { @@ -20,14 +20,16 @@ func Test(t *testing.T) { return } defer func() { - os.RemoveAll(dir) + if err := os.RemoveAll(dir); err != nil { + t.Fatal(err) + } }() - fmt.Printf("DB path : %s\n", filepath.Join(dir, "ql.db")) + t.Logf("DB path : %s\n", filepath.Join(dir, "ql.db")) p := &Ql{} addr := fmt.Sprintf("ql://%s", filepath.Join(dir, "ql.db")) d, err := p.Open(addr) if err != nil { - t.Fatalf("%v", err) + t.Fatal(err) } db, err := sql.Open("ql", filepath.Join(dir, "ql.db")) @@ -40,23 +42,40 @@ func Test(t *testing.T) { } }() dt.Test(t, d, []byte("CREATE TABLE t (Qty int, Name string);")) - driver, err := WithInstance(db, &Config{}) +} + +func TestMigrate(t *testing.T) { + dir, err := ioutil.TempDir("", "ql-driver-test") + if err != nil { + return + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + }() + t.Logf("DB path : %s\n", filepath.Join(dir, "ql.db")) + + db, err := sql.Open("ql", filepath.Join(dir, "ql.db")) if err != nil { - t.Fatalf("%v", err) + return } - if err := d.Drop(); err != nil { + defer func() { + if err := db.Close(); err != nil { + return + } + }() + + driver, err := WithInstance(db, &Config{}) + if err != nil { t.Fatal(err) } m, err := migrate.NewWithDatabaseInstance( - "file://./migration", + "file://./examples/migrations", "ql", driver) if err != nil { - t.Fatalf("%v", err) - } - fmt.Println("UP") - err = m.Up() - if err != nil { - t.Fatalf("%v", err) + t.Fatal(err) } + dt.TestMigrate(t, m) } diff --git a/database/redshift/README.md b/database/redshift/README.md index a03d109ae..52f9a5639 100644 --- a/database/redshift/README.md +++ b/database/redshift/README.md @@ -1,6 +1,21 @@ -Redshift -=== +# Redshift -This provides a Redshift driver for migrations. It is used whenever the URL of the database starts with `redshift://`. +`redshift://user:password@host:port/dbname?query` + +| URL Query | WithInstance Config | Description | +|------------|---------------------|-------------| +| `x-migrations-table` | `MigrationsTable` | Name of the migrations table | +| `dbname` | `DatabaseName` | The name of the database to connect to | +| `search_path` | | This variable specifies the order in which schemas are searched when an object is referenced by a simple name with no schema specified. | +| `user` | | The user to sign in as | +| `password` | | The user's password | +| `host` | | The host to connect to. Values that start with / are for unix domain sockets. (default is localhost) | +| `port` | | The port to bind to. (default is 5439) | +| `fallback_application_name` | | An application_name to fall back to if one isn't provided. | +| `connect_timeout` | | Maximum wait for connection, in seconds. Zero or not specified means wait indefinitely. | +| `sslcert` | | Cert file location. The file must contain PEM encoded data. | +| `sslkey` | | Key file location. The file must contain PEM encoded data. | +| `sslrootcert` | | The location of the root certificate file. The file must contain PEM encoded data. | +| `sslmode` | | Whether or not to use SSL (disable\|require\|verify-ca\|verify-full) | Redshift is PostgreSQL compatible but has some specific features (or lack thereof) that require slightly different behavior. diff --git a/database/redshift/examples/migrations/1085649617_create_users_table.down.sql b/database/redshift/examples/migrations/1085649617_create_users_table.down.sql new file mode 100644 index 000000000..c99ddcdc8 --- /dev/null +++ b/database/redshift/examples/migrations/1085649617_create_users_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS users; diff --git a/database/redshift/examples/migrations/1085649617_create_users_table.up.sql b/database/redshift/examples/migrations/1085649617_create_users_table.up.sql new file mode 100644 index 000000000..92897dcab --- /dev/null +++ b/database/redshift/examples/migrations/1085649617_create_users_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE users ( + user_id integer unique, + name varchar(40), + email varchar(40) +); diff --git a/database/redshift/examples/migrations/1185749658_add_city_to_users.down.sql b/database/redshift/examples/migrations/1185749658_add_city_to_users.down.sql new file mode 100644 index 000000000..940c60712 --- /dev/null +++ b/database/redshift/examples/migrations/1185749658_add_city_to_users.down.sql @@ -0,0 +1 @@ +ALTER TABLE users DROP COLUMN IF EXISTS city; diff --git a/database/redshift/examples/migrations/1185749658_add_city_to_users.up.sql b/database/redshift/examples/migrations/1185749658_add_city_to_users.up.sql new file mode 100644 index 000000000..67823edc9 --- /dev/null +++ b/database/redshift/examples/migrations/1185749658_add_city_to_users.up.sql @@ -0,0 +1,3 @@ +ALTER TABLE users ADD COLUMN city varchar(100); + + diff --git a/database/redshift/examples/migrations/1285849751_add_index_on_user_emails.down.sql b/database/redshift/examples/migrations/1285849751_add_index_on_user_emails.down.sql new file mode 100644 index 000000000..3e87dd229 --- /dev/null +++ b/database/redshift/examples/migrations/1285849751_add_index_on_user_emails.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS users_email_index; diff --git a/database/redshift/examples/migrations/1285849751_add_index_on_user_emails.up.sql b/database/redshift/examples/migrations/1285849751_add_index_on_user_emails.up.sql new file mode 100644 index 000000000..fbeb4ab4e --- /dev/null +++ b/database/redshift/examples/migrations/1285849751_add_index_on_user_emails.up.sql @@ -0,0 +1,3 @@ +CREATE UNIQUE INDEX CONCURRENTLY users_email_index ON users (email); + +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/redshift/examples/migrations/1385949617_create_books_table.down.sql b/database/redshift/examples/migrations/1385949617_create_books_table.down.sql new file mode 100644 index 000000000..1a0b1a214 --- /dev/null +++ b/database/redshift/examples/migrations/1385949617_create_books_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS books; diff --git a/database/redshift/examples/migrations/1385949617_create_books_table.up.sql b/database/redshift/examples/migrations/1385949617_create_books_table.up.sql new file mode 100644 index 000000000..f1503b518 --- /dev/null +++ b/database/redshift/examples/migrations/1385949617_create_books_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE books ( + user_id integer, + name varchar(40), + author varchar(40) +); diff --git a/database/redshift/examples/migrations/1485949617_create_movies_table.down.sql b/database/redshift/examples/migrations/1485949617_create_movies_table.down.sql new file mode 100644 index 000000000..3a5187689 --- /dev/null +++ b/database/redshift/examples/migrations/1485949617_create_movies_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS movies; diff --git a/database/redshift/examples/migrations/1485949617_create_movies_table.up.sql b/database/redshift/examples/migrations/1485949617_create_movies_table.up.sql new file mode 100644 index 000000000..f0ef5943b --- /dev/null +++ b/database/redshift/examples/migrations/1485949617_create_movies_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE movies ( + user_id integer, + name varchar(40), + director varchar(40) +); diff --git a/database/redshift/examples/migrations/1585849751_just_a_comment.up.sql b/database/redshift/examples/migrations/1585849751_just_a_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/redshift/examples/migrations/1585849751_just_a_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/redshift/examples/migrations/1685849751_another_comment.up.sql b/database/redshift/examples/migrations/1685849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/redshift/examples/migrations/1685849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/redshift/examples/migrations/1785849751_another_comment.up.sql b/database/redshift/examples/migrations/1785849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/redshift/examples/migrations/1785849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/redshift/examples/migrations/1885849751_another_comment.up.sql b/database/redshift/examples/migrations/1885849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/redshift/examples/migrations/1885849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/redshift/redshift.go b/database/redshift/redshift.go index 96001f964..04bee121d 100644 --- a/database/redshift/redshift.go +++ b/database/redshift/redshift.go @@ -1,46 +1,341 @@ +//go:build go1.9 +// +build go1.9 + package redshift import ( - "net/url" + "context" + "database/sql" + "fmt" + "go.uber.org/atomic" + "io" + "io/ioutil" + nurl "net/url" + "strconv" + "strings" + "github.com/golang-migrate/migrate/v4" "github.com/golang-migrate/migrate/v4/database" - "github.com/golang-migrate/migrate/v4/database/postgres" + "github.com/hashicorp/go-multierror" + "github.com/lib/pq" ) -// init registers the driver under the name 'redshift' func init() { - db := new(Redshift) - db.Driver = new(postgres.Postgres) + db := Redshift{} + database.Register("redshift", &db) +} + +var DefaultMigrationsTable = "schema_migrations" - database.Register("redshift", db) +var ( + ErrNilConfig = fmt.Errorf("no config") + ErrNoDatabaseName = fmt.Errorf("no database name") +) + +type Config struct { + MigrationsTable string + DatabaseName string } -// Redshift is a wrapper around the PostgreSQL driver which implements Redshift-specific behavior. -// -// Currently, the only different behaviour is the lack of locking in Redshift. The (Un)Lock() method(s) have been overridden from the PostgreSQL adapter to simply return nil. type Redshift struct { - // The wrapped PostgreSQL driver. - database.Driver + isLocked atomic.Bool + conn *sql.Conn + db *sql.DB + + // Open and WithInstance need to guarantee that config is never nil + config *Config } -// Open implements the database.Driver interface by parsing the URL, switching the scheme from "redshift" to "postgres", and delegating to the underlying PostgreSQL driver. -func (driver *Redshift) Open(dsn string) (database.Driver, error) { - parsed, err := url.Parse(dsn) +func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { + if config == nil { + return nil, ErrNilConfig + } + + if err := instance.Ping(); err != nil { + return nil, err + } + + if config.DatabaseName == "" { + query := `SELECT CURRENT_DATABASE()` + var databaseName string + if err := instance.QueryRow(query).Scan(&databaseName); err != nil { + return nil, &database.Error{OrigErr: err, Query: []byte(query)} + } + + if len(databaseName) == 0 { + return nil, ErrNoDatabaseName + } + + config.DatabaseName = databaseName + } + + if len(config.MigrationsTable) == 0 { + config.MigrationsTable = DefaultMigrationsTable + } + + conn, err := instance.Conn(context.Background()) + if err != nil { return nil, err } - parsed.Scheme = "postgres" - psql, err := driver.Driver.Open(parsed.String()) + px := &Redshift{ + conn: conn, + db: instance, + config: config, + } + + if err := px.ensureVersionTable(); err != nil { + return nil, err + } + + return px, nil +} + +func (p *Redshift) Open(url string) (database.Driver, error) { + purl, err := nurl.Parse(url) + if err != nil { + return nil, err + } + purl.Scheme = "postgres" + + db, err := sql.Open("postgres", migrate.FilterCustomQuery(purl).String()) + if err != nil { + return nil, err + } + + migrationsTable := purl.Query().Get("x-migrations-table") + + px, err := WithInstance(db, &Config{ + DatabaseName: purl.Path, + MigrationsTable: migrationsTable, + }) if err != nil { return nil, err } - return &Redshift{Driver: psql}, nil + return px, nil } -// Lock implements the database.Driver interface by not locking and returning nil. -func (driver *Redshift) Lock() error { return nil } +func (p *Redshift) Close() error { + connErr := p.conn.Close() + dbErr := p.db.Close() + if connErr != nil || dbErr != nil { + return fmt.Errorf("conn: %v, db: %v", connErr, dbErr) + } + return nil +} + +// Redshift does not support advisory lock functions: https://docs.aws.amazon.com/redshift/latest/dg/c_unsupported-postgresql-functions.html +func (p *Redshift) Lock() error { + if !p.isLocked.CAS(false, true) { + return database.ErrLocked + } + return nil +} + +func (p *Redshift) Unlock() error { + if !p.isLocked.CAS(true, false) { + return database.ErrNotLocked + } + return nil +} + +func (p *Redshift) Run(migration io.Reader) error { + migr, err := ioutil.ReadAll(migration) + if err != nil { + return err + } + + // run migration + query := string(migr[:]) + if _, err := p.conn.ExecContext(context.Background(), query); err != nil { + if pgErr, ok := err.(*pq.Error); ok { + var line uint + var col uint + var lineColOK bool + if pgErr.Position != "" { + if pos, err := strconv.ParseUint(pgErr.Position, 10, 64); err == nil { + line, col, lineColOK = computeLineFromPos(query, int(pos)) + } + } + message := fmt.Sprintf("migration failed: %s", pgErr.Message) + if lineColOK { + message = fmt.Sprintf("%s (column %d)", message, col) + } + if pgErr.Detail != "" { + message = fmt.Sprintf("%s, %s", message, pgErr.Detail) + } + return database.Error{OrigErr: err, Err: message, Query: migr, Line: line} + } + return database.Error{OrigErr: err, Err: "migration failed", Query: migr} + } -// Unlock implements the database.Driver interface by not unlocking and returning nil. -func (driver *Redshift) Unlock() error { return nil } + return nil +} + +func computeLineFromPos(s string, pos int) (line uint, col uint, ok bool) { + // replace crlf with lf + s = strings.Replace(s, "\r\n", "\n", -1) + // pg docs: pos uses index 1 for the first character, and positions are measured in characters not bytes + runes := []rune(s) + if pos > len(runes) { + return 0, 0, false + } + sel := runes[:pos] + line = uint(runesCount(sel, newLine) + 1) + col = uint(pos - 1 - runesLastIndex(sel, newLine)) + return line, col, true +} + +const newLine = '\n' + +func runesCount(input []rune, target rune) int { + var count int + for _, r := range input { + if r == target { + count++ + } + } + return count +} + +func runesLastIndex(input []rune, target rune) int { + for i := len(input) - 1; i >= 0; i-- { + if input[i] == target { + return i + } + } + return -1 +} + +func (p *Redshift) SetVersion(version int, dirty bool) error { + tx, err := p.conn.BeginTx(context.Background(), &sql.TxOptions{}) + if err != nil { + return &database.Error{OrigErr: err, Err: "transaction start failed"} + } + + query := `DELETE FROM "` + p.config.MigrationsTable + `"` + if _, err := tx.Exec(query); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + // Also re-write the schema version for nil dirty versions to prevent + // empty schema version for failed down migration on the first migration + // See: https://github.com/golang-migrate/migrate/issues/330 + if version >= 0 || (version == database.NilVersion && dirty) { + query = `INSERT INTO "` + p.config.MigrationsTable + `" (version, dirty) VALUES ($1, $2)` + if _, err := tx.Exec(query, version, dirty); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + + if err := tx.Commit(); err != nil { + return &database.Error{OrigErr: err, Err: "transaction commit failed"} + } + + return nil +} + +func (p *Redshift) Version() (version int, dirty bool, err error) { + query := `SELECT version, dirty FROM "` + p.config.MigrationsTable + `" LIMIT 1` + err = p.conn.QueryRowContext(context.Background(), query).Scan(&version, &dirty) + switch { + case err == sql.ErrNoRows: + return database.NilVersion, false, nil + + case err != nil: + if e, ok := err.(*pq.Error); ok { + if e.Code.Name() == "undefined_table" { + return database.NilVersion, false, nil + } + } + return 0, false, &database.Error{OrigErr: err, Query: []byte(query)} + + default: + return version, dirty, nil + } +} + +func (p *Redshift) Drop() (err error) { + // select all tables in current schema + query := `SELECT table_name FROM information_schema.tables WHERE table_schema=(SELECT current_schema()) AND table_type='BASE TABLE'` + tables, err := p.conn.QueryContext(context.Background(), query) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + defer func() { + if errClose := tables.Close(); errClose != nil { + err = multierror.Append(err, errClose) + } + }() + + // delete one table after another + tableNames := make([]string, 0) + for tables.Next() { + var tableName string + if err := tables.Scan(&tableName); err != nil { + return err + } + if len(tableName) > 0 { + tableNames = append(tableNames, tableName) + } + } + if err := tables.Err(); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + if len(tableNames) > 0 { + // delete one by one ... + for _, t := range tableNames { + query = `DROP TABLE IF EXISTS ` + t + ` CASCADE` + if _, err := p.conn.ExecContext(context.Background(), query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + } + + return nil +} + +// ensureVersionTable checks if versions table exists and, if not, creates it. +// Note that this function locks the database, which deviates from the usual +// convention of "caller locks" in the Redshift type. +func (p *Redshift) ensureVersionTable() (err error) { + if err = p.Lock(); err != nil { + return err + } + + defer func() { + if e := p.Unlock(); e != nil { + if err == nil { + err = e + } else { + err = multierror.Append(err, e) + } + } + }() + + // check if migration table exists + var count int + query := `SELECT COUNT(1) FROM information_schema.tables WHERE table_name = $1 AND table_schema = (SELECT current_schema()) LIMIT 1` + if err := p.conn.QueryRowContext(context.Background(), query, p.config.MigrationsTable).Scan(&count); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + if count == 1 { + return nil + } + + // if not, create the empty migration table + query = `CREATE TABLE "` + p.config.MigrationsTable + `" (version bigint not null primary key, dirty boolean not null)` + if _, err := p.conn.ExecContext(context.Background(), query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + return nil +} diff --git a/database/redshift/redshift_test.go b/database/redshift/redshift_test.go new file mode 100644 index 000000000..944a6add3 --- /dev/null +++ b/database/redshift/redshift_test.go @@ -0,0 +1,401 @@ +package redshift + +// error codes https://github.com/lib/pq/blob/master/error.go + +import ( + "bytes" + "context" + "database/sql" + sqldriver "database/sql/driver" + "fmt" + "log" + + "github.com/golang-migrate/migrate/v4" + "io" + "strconv" + "strings" + "testing" +) + +import ( + "github.com/dhui/dktest" +) + +import ( + "github.com/golang-migrate/migrate/v4/database" + dt "github.com/golang-migrate/migrate/v4/database/testing" + "github.com/golang-migrate/migrate/v4/dktesting" + _ "github.com/golang-migrate/migrate/v4/source/file" +) + +var ( + opts = dktest.Options{PortRequired: true, ReadyFunc: isReady} + specs = []dktesting.ContainerSpec{ + {ImageName: "postgres:8", Options: opts}, + } +) + +func redshiftConnectionString(host, port string) string { + return connectionString("redshift", host, port) +} + +func pgConnectionString(host, port string) string { + return connectionString("postgres", host, port) +} + +func connectionString(schema, host, port string) string { + return fmt.Sprintf("%s://postgres@%s:%s/postgres?sslmode=disable", schema, host, port) +} + +func isReady(ctx context.Context, c dktest.ContainerInfo) bool { + ip, port, err := c.FirstPort() + if err != nil { + return false + } + + db, err := sql.Open("postgres", pgConnectionString(ip, port)) + if err != nil { + return false + } + defer func() { + if err := db.Close(); err != nil { + log.Println("close error:", err) + } + }() + if err = db.PingContext(ctx); err != nil { + switch err { + case sqldriver.ErrBadConn, io.EOF: + return false + default: + log.Println(err) + } + return false + } + + return true +} + +func Test(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := redshiftConnectionString(ip, port) + p := &Redshift{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + dt.Test(t, d, []byte("SELECT 1")) + }) +} + +func TestMigrate(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := redshiftConnectionString(ip, port) + p := &Redshift{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "postgres", d) + if err != nil { + t.Fatal(err) + } + dt.TestMigrate(t, m) + }) +} + +func TestMultiStatement(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := redshiftConnectionString(ip, port) + p := &Redshift{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + if err := d.Run(bytes.NewReader([]byte("CREATE TABLE foo (foo text); CREATE TABLE bar (bar text);"))); err != nil { + t.Fatalf("expected err to be nil, got %v", err) + } + + // make sure second table exists + var exists bool + if err := d.(*Redshift).conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'bar' AND table_schema = (SELECT current_schema()))").Scan(&exists); err != nil { + t.Fatal(err) + } + if !exists { + t.Fatalf("expected table bar to exist") + } + }) +} + +func TestErrorParsing(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := redshiftConnectionString(ip, port) + p := &Redshift{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + + wantErr := `migration failed: syntax error at or near "TABLEE" (column 37) in line 1: CREATE TABLE foo ` + + `(foo text); CREATE TABLEE bar (bar text); (details: pq: syntax error at or near "TABLEE")` + if err := d.Run(bytes.NewReader([]byte("CREATE TABLE foo (foo text); CREATE TABLEE bar (bar text);"))); err == nil { + t.Fatal("expected err but got nil") + } else if err.Error() != wantErr { + t.Fatalf("expected '%s' but got '%s'", wantErr, err.Error()) + } + }) +} + +func TestFilterCustomQuery(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := fmt.Sprintf("postgres://postgres@%v:%v/postgres?sslmode=disable&x-custom=foobar", ip, port) + p := &Redshift{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + }) +} + +func TestWithSchema(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := redshiftConnectionString(ip, port) + p := &Redshift{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + + // create foobar schema + if err := d.Run(bytes.NewReader([]byte("CREATE SCHEMA foobar AUTHORIZATION postgres"))); err != nil { + t.Fatal(err) + } + if err := d.SetVersion(1, false); err != nil { + t.Fatal(err) + } + + // re-connect using that schema + d2, err := p.Open(fmt.Sprintf("postgres://postgres@%v:%v/postgres?sslmode=disable&search_path=foobar", ip, port)) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d2.Close(); err != nil { + t.Error(err) + } + }() + + version, _, err := d2.Version() + if err != nil { + t.Fatal(err) + } + if version != database.NilVersion { + t.Fatal("expected NilVersion") + } + + // now update version and compare + if err := d2.SetVersion(2, false); err != nil { + t.Fatal(err) + } + version, _, err = d2.Version() + if err != nil { + t.Fatal(err) + } + if version != 2 { + t.Fatal("expected version 2") + } + + // meanwhile, the public schema still has the other version + version, _, err = d.Version() + if err != nil { + t.Fatal(err) + } + if version != 1 { + t.Fatal("expected version 2") + } + }) +} + +func TestWithInstance(t *testing.T) { + +} + +func TestRedshift_Lock(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := pgConnectionString(ip, port) + p := &Redshift{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + + dt.Test(t, d, []byte("SELECT 1")) + + ps := d.(*Redshift) + + err = ps.Lock() + if err != nil { + t.Fatal(err) + } + + err = ps.Unlock() + if err != nil { + t.Fatal(err) + } + + err = ps.Lock() + if err != nil { + t.Fatal(err) + } + + err = ps.Unlock() + if err != nil { + t.Fatal(err) + } + }) +} + +func Test_computeLineFromPos(t *testing.T) { + testcases := []struct { + pos int + wantLine uint + wantCol uint + input string + wantOk bool + }{ + { + 15, 2, 6, "SELECT *\nFROM foo", true, // foo table does not exists + }, + { + 16, 3, 6, "SELECT *\n\nFROM foo", true, // foo table does not exists, empty line + }, + { + 25, 3, 7, "SELECT *\nFROM foo\nWHERE x", true, // x column error + }, + { + 27, 5, 7, "SELECT *\n\nFROM foo\n\nWHERE x", true, // x column error, empty lines + }, + { + 10, 2, 1, "SELECT *\nFROMM foo", true, // FROMM typo + }, + { + 11, 3, 1, "SELECT *\n\nFROMM foo", true, // FROMM typo, empty line + }, + { + 17, 2, 8, "SELECT *\nFROM foo", true, // last character + }, + { + 18, 0, 0, "SELECT *\nFROM foo", false, // invalid position + }, + } + for i, tc := range testcases { + t.Run("tc"+strconv.Itoa(i), func(t *testing.T) { + run := func(crlf bool, nonASCII bool) { + var name string + if crlf { + name = "crlf" + } else { + name = "lf" + } + if nonASCII { + name += "-nonascii" + } else { + name += "-ascii" + } + t.Run(name, func(t *testing.T) { + input := tc.input + if crlf { + input = strings.Replace(input, "\n", "\r\n", -1) + } + if nonASCII { + input = strings.Replace(input, "FROM", "FRÖM", -1) + } + gotLine, gotCol, gotOK := computeLineFromPos(input, tc.pos) + + if tc.wantOk { + t.Logf("pos %d, want %d:%d, %#v", tc.pos, tc.wantLine, tc.wantCol, input) + } + + if gotOK != tc.wantOk { + t.Fatalf("expected ok %v but got %v", tc.wantOk, gotOK) + } + if gotLine != tc.wantLine { + t.Fatalf("expected line %d but got %d", tc.wantLine, gotLine) + } + if gotCol != tc.wantCol { + t.Fatalf("expected col %d but got %d", tc.wantCol, gotCol) + } + }) + } + run(false, false) + run(true, false) + run(false, true) + run(true, true) + }) + } + +} diff --git a/database/snowflake/README.md b/database/snowflake/README.md new file mode 100644 index 000000000..90a28d177 --- /dev/null +++ b/database/snowflake/README.md @@ -0,0 +1,12 @@ +# Snowflake + +`snowflake://user:password@accountname/schema/dbname?query` + +| URL Query | WithInstance Config | Description | +|------------|---------------------|-------------| +| `x-migrations-table` | `MigrationsTable` | Name of the migrations table | + +Snowflake is PostgreSQL compatible but has some specific features (or lack thereof) that require slightly different behavior. + +## Status +This driver is not officially supported as there are no tests for it. diff --git a/database/snowflake/snowflake.go b/database/snowflake/snowflake.go new file mode 100644 index 000000000..53d7ca282 --- /dev/null +++ b/database/snowflake/snowflake.go @@ -0,0 +1,376 @@ +package snowflake + +import ( + "context" + "database/sql" + "fmt" + "go.uber.org/atomic" + "io" + "io/ioutil" + nurl "net/url" + "strconv" + "strings" + + "github.com/golang-migrate/migrate/v4/database" + "github.com/hashicorp/go-multierror" + "github.com/lib/pq" + sf "github.com/snowflakedb/gosnowflake" +) + +func init() { + db := Snowflake{} + database.Register("snowflake", &db) +} + +var DefaultMigrationsTable = "schema_migrations" + +var ( + ErrNilConfig = fmt.Errorf("no config") + ErrNoDatabaseName = fmt.Errorf("no database name") + ErrNoPassword = fmt.Errorf("no password") + ErrNoSchema = fmt.Errorf("no schema") + ErrNoSchemaOrDatabase = fmt.Errorf("no schema/database name") +) + +type Config struct { + MigrationsTable string + DatabaseName string +} + +type Snowflake struct { + isLocked atomic.Bool + conn *sql.Conn + db *sql.DB + + // Open and WithInstance need to guarantee that config is never nil + config *Config +} + +func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { + if config == nil { + return nil, ErrNilConfig + } + + if err := instance.Ping(); err != nil { + return nil, err + } + + if config.DatabaseName == "" { + query := `SELECT CURRENT_DATABASE()` + var databaseName string + if err := instance.QueryRow(query).Scan(&databaseName); err != nil { + return nil, &database.Error{OrigErr: err, Query: []byte(query)} + } + + if len(databaseName) == 0 { + return nil, ErrNoDatabaseName + } + + config.DatabaseName = databaseName + } + + if len(config.MigrationsTable) == 0 { + config.MigrationsTable = DefaultMigrationsTable + } + + conn, err := instance.Conn(context.Background()) + + if err != nil { + return nil, err + } + + px := &Snowflake{ + conn: conn, + db: instance, + config: config, + } + + if err := px.ensureVersionTable(); err != nil { + return nil, err + } + + return px, nil +} + +func (p *Snowflake) Open(url string) (database.Driver, error) { + purl, err := nurl.Parse(url) + if err != nil { + return nil, err + } + + password, isPasswordSet := purl.User.Password() + if !isPasswordSet { + return nil, ErrNoPassword + } + + splitPath := strings.Split(purl.Path, "/") + if len(splitPath) < 3 { + return nil, ErrNoSchemaOrDatabase + } + + database := splitPath[2] + if len(database) == 0 { + return nil, ErrNoDatabaseName + } + + schema := splitPath[1] + if len(schema) == 0 { + return nil, ErrNoSchema + } + + cfg := &sf.Config{ + Account: purl.Host, + User: purl.User.Username(), + Password: password, + Database: database, + Schema: schema, + } + + dsn, err := sf.DSN(cfg) + if err != nil { + return nil, err + } + + db, err := sql.Open("snowflake", dsn) + if err != nil { + return nil, err + } + + migrationsTable := purl.Query().Get("x-migrations-table") + + px, err := WithInstance(db, &Config{ + DatabaseName: database, + MigrationsTable: migrationsTable, + }) + if err != nil { + return nil, err + } + + return px, nil +} + +func (p *Snowflake) Close() error { + connErr := p.conn.Close() + dbErr := p.db.Close() + if connErr != nil || dbErr != nil { + return fmt.Errorf("conn: %v, db: %v", connErr, dbErr) + } + return nil +} + +func (p *Snowflake) Lock() error { + if !p.isLocked.CAS(false, true) { + return database.ErrLocked + } + return nil +} + +func (p *Snowflake) Unlock() error { + if !p.isLocked.CAS(true, false) { + return database.ErrNotLocked + } + return nil +} + +func (p *Snowflake) Run(migration io.Reader) error { + migr, err := ioutil.ReadAll(migration) + if err != nil { + return err + } + + // run migration + query := string(migr[:]) + if _, err := p.conn.ExecContext(context.Background(), query); err != nil { + if pgErr, ok := err.(*pq.Error); ok { + var line uint + var col uint + var lineColOK bool + if pgErr.Position != "" { + if pos, err := strconv.ParseUint(pgErr.Position, 10, 64); err == nil { + line, col, lineColOK = computeLineFromPos(query, int(pos)) + } + } + message := fmt.Sprintf("migration failed: %s", pgErr.Message) + if lineColOK { + message = fmt.Sprintf("%s (column %d)", message, col) + } + if pgErr.Detail != "" { + message = fmt.Sprintf("%s, %s", message, pgErr.Detail) + } + return database.Error{OrigErr: err, Err: message, Query: migr, Line: line} + } + return database.Error{OrigErr: err, Err: "migration failed", Query: migr} + } + + return nil +} + +func computeLineFromPos(s string, pos int) (line uint, col uint, ok bool) { + // replace crlf with lf + s = strings.Replace(s, "\r\n", "\n", -1) + // pg docs: pos uses index 1 for the first character, and positions are measured in characters not bytes + runes := []rune(s) + if pos > len(runes) { + return 0, 0, false + } + sel := runes[:pos] + line = uint(runesCount(sel, newLine) + 1) + col = uint(pos - 1 - runesLastIndex(sel, newLine)) + return line, col, true +} + +const newLine = '\n' + +func runesCount(input []rune, target rune) int { + var count int + for _, r := range input { + if r == target { + count++ + } + } + return count +} + +func runesLastIndex(input []rune, target rune) int { + for i := len(input) - 1; i >= 0; i-- { + if input[i] == target { + return i + } + } + return -1 +} + +func (p *Snowflake) SetVersion(version int, dirty bool) error { + tx, err := p.conn.BeginTx(context.Background(), &sql.TxOptions{}) + if err != nil { + return &database.Error{OrigErr: err, Err: "transaction start failed"} + } + + query := `DELETE FROM "` + p.config.MigrationsTable + `"` + if _, err := tx.Exec(query); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + // Also re-write the schema version for nil dirty versions to prevent + // empty schema version for failed down migration on the first migration + // See: https://github.com/golang-migrate/migrate/issues/330 + if version >= 0 || (version == database.NilVersion && dirty) { + query = `INSERT INTO "` + p.config.MigrationsTable + `" (version, + dirty) VALUES (` + strconv.FormatInt(int64(version), 10) + `, + ` + strconv.FormatBool(dirty) + `)` + if _, err := tx.Exec(query); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + + if err := tx.Commit(); err != nil { + return &database.Error{OrigErr: err, Err: "transaction commit failed"} + } + + return nil +} + +func (p *Snowflake) Version() (version int, dirty bool, err error) { + query := `SELECT version, dirty FROM "` + p.config.MigrationsTable + `" LIMIT 1` + err = p.conn.QueryRowContext(context.Background(), query).Scan(&version, &dirty) + switch { + case err == sql.ErrNoRows: + return database.NilVersion, false, nil + + case err != nil: + if e, ok := err.(*pq.Error); ok { + if e.Code.Name() == "undefined_table" { + return database.NilVersion, false, nil + } + } + return 0, false, &database.Error{OrigErr: err, Query: []byte(query)} + + default: + return version, dirty, nil + } +} + +func (p *Snowflake) Drop() (err error) { + // select all tables in current schema + query := `SELECT table_name FROM information_schema.tables WHERE table_schema=(SELECT current_schema()) AND table_type='BASE TABLE'` + tables, err := p.conn.QueryContext(context.Background(), query) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + defer func() { + if errClose := tables.Close(); errClose != nil { + err = multierror.Append(err, errClose) + } + }() + + // delete one table after another + tableNames := make([]string, 0) + for tables.Next() { + var tableName string + if err := tables.Scan(&tableName); err != nil { + return err + } + if len(tableName) > 0 { + tableNames = append(tableNames, tableName) + } + } + if err := tables.Err(); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + if len(tableNames) > 0 { + // delete one by one ... + for _, t := range tableNames { + query = `DROP TABLE IF EXISTS ` + t + ` CASCADE` + if _, err := p.conn.ExecContext(context.Background(), query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + } + + return nil +} + +// ensureVersionTable checks if versions table exists and, if not, creates it. +// Note that this function locks the database, which deviates from the usual +// convention of "caller locks" in the Snowflake type. +func (p *Snowflake) ensureVersionTable() (err error) { + if err = p.Lock(); err != nil { + return err + } + + defer func() { + if e := p.Unlock(); e != nil { + if err == nil { + err = e + } else { + err = multierror.Append(err, e) + } + } + }() + + // check if migration table exists + var count int + query := `SELECT COUNT(1) FROM information_schema.tables WHERE table_name = $1 AND table_schema = (SELECT current_schema()) LIMIT 1` + if err := p.conn.QueryRowContext(context.Background(), query, p.config.MigrationsTable).Scan(&count); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + if count == 1 { + return nil + } + + // if not, create the empty migration table + query = `CREATE TABLE if not exists "` + p.config.MigrationsTable + `" ( + version bigint not null primary key, dirty boolean not null)` + if _, err := p.conn.ExecContext(context.Background(), query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + return nil +} diff --git a/database/spanner/README.md b/database/spanner/README.md index 0de867a8d..75ece73f1 100644 --- a/database/spanner/README.md +++ b/database/spanner/README.md @@ -2,22 +2,24 @@ ## Usage -The DSN must be given in the following format. +See [Google Spanner Documentation](https://cloud.google.com/spanner/docs) for +more details. -`spanner://projects/{projectId}/instances/{instanceId}/databases/{databaseName}` +The DSN must be given in the following format. -See [Google Spanner Documentation](https://cloud.google.com/spanner/docs) for details. +`spanner://projects/{projectId}/instances/{instanceId}/databases/{databaseName}?param=true` +as described in [README.md#database-urls](../../README.md#database-urls) | Param | WithInstance Config | Description | | ----- | ------------------- | ----------- | | `x-migrations-table` | `MigrationsTable` | Name of the migrations table | +| `x-clean-statements` | `CleanStatements` | Whether to parse and clean DDL statements before running migration towards Spanner (Required for comments and multiple statements) | | `url` | `DatabaseName` | The full path to the Spanner database resource. If provided as part of `Config` it must not contain a scheme or query string to match the format `projects/{projectId}/instances/{instanceId}/databases/{databaseName}`| | `projectId` || The Google Cloud Platform project id | `instanceId` || The id of the instance running Spanner | `databaseName` || The name of the Spanner database - > **Note:** Google Cloud Spanner migrations can take a considerable amount of > time. The migrations provided as part of the example take about 6 minutes to > run on a small instance. @@ -28,8 +30,17 @@ See [Google Spanner Documentation](https://cloud.google.com/spanner/docs) for de > 1496601752/u add_index_on_user_emails (2m12.155787369s) > 1496602638/u create_books_table (2m30.77299181s) +## DDL with comments + +At the moment the GCP Spanner backed does not seem to allow for comments (See https://issuetracker.google.com/issues/159730604) +so in order to be able to use migration with DDL containing comments `x-clean-stamements` is required + +## Multiple statements + +In order to be able to use more than 1 DDL statement in the same migration file, the file has to be parsed and therefore the `x-clean-statements` flag is required + ## Testing To unit test the `spanner` driver, `SPANNER_DATABASE` needs to be set. You'll need to sign-up to Google Cloud Platform (GCP) and have a running Spanner -instance since it is not possible to run Google Spanner outside GCP. \ No newline at end of file +instance since it is not possible to run Google Spanner outside GCP. diff --git a/database/spanner/examples/migrations/1621360367_create_transactions_table.down.sql b/database/spanner/examples/migrations/1621360367_create_transactions_table.down.sql new file mode 100644 index 000000000..1d1700a95 --- /dev/null +++ b/database/spanner/examples/migrations/1621360367_create_transactions_table.down.sql @@ -0,0 +1 @@ +DROP TABLE Transactions diff --git a/database/spanner/examples/migrations/1621360367_create_transactions_table.up.sql b/database/spanner/examples/migrations/1621360367_create_transactions_table.up.sql new file mode 100644 index 000000000..59d952eb9 --- /dev/null +++ b/database/spanner/examples/migrations/1621360367_create_transactions_table.up.sql @@ -0,0 +1,6 @@ +CREATE TABLE Transactions ( + UserId INT64, + TransactionId STRING(40), + Total NUMERIC +) PRIMARY KEY(UserId, TransactionId), +INTERLEAVE IN PARENT Users ON DELETE CASCADE diff --git a/database/spanner/spanner.go b/database/spanner/spanner.go index f59046a77..16f0f9d28 100644 --- a/database/spanner/spanner.go +++ b/database/spanner/spanner.go @@ -1,22 +1,27 @@ package spanner import ( + "errors" "fmt" "io" "io/ioutil" "log" nurl "net/url" "regexp" + "strconv" "strings" - "golang.org/x/net/context" + "context" "cloud.google.com/go/spanner" sdb "cloud.google.com/go/spanner/admin/database/apiv1" + "cloud.google.com/go/spanner/spansql" "github.com/golang-migrate/migrate/v4" "github.com/golang-migrate/migrate/v4/database" + "github.com/hashicorp/go-multierror" + uatomic "go.uber.org/atomic" "google.golang.org/api/iterator" adminpb "google.golang.org/genproto/googleapis/spanner/admin/database/v1" ) @@ -29,18 +34,30 @@ func init() { // DefaultMigrationsTable is used if no custom table is specified const DefaultMigrationsTable = "SchemaMigrations" +const ( + unlockedVal = 0 + lockedVal = 1 +) + // Driver errors var ( - ErrNilConfig = fmt.Errorf("no config") - ErrNoDatabaseName = fmt.Errorf("no database name") - ErrNoSchema = fmt.Errorf("no schema") - ErrDatabaseDirty = fmt.Errorf("database is dirty") + ErrNilConfig = errors.New("no config") + ErrNoDatabaseName = errors.New("no database name") + ErrNoSchema = errors.New("no schema") + ErrDatabaseDirty = errors.New("database is dirty") + ErrLockHeld = errors.New("unable to obtain lock") + ErrLockNotHeld = errors.New("unable to release already released lock") ) // Config used for a Spanner instance type Config struct { MigrationsTable string DatabaseName string + // Whether to parse the migration DDL with spansql before + // running them towards Spanner. + // Parsing outputs clean DDL statements such as reformatted + // and void of comments. + CleanStatements bool } // Spanner implements database.Driver for Google Cloud Spanner @@ -48,6 +65,8 @@ type Spanner struct { db *DB config *Config + + lock *uatomic.Uint32 } type DB struct { @@ -55,6 +74,13 @@ type DB struct { data *spanner.Client } +func NewDB(admin sdb.DatabaseAdminClient, data spanner.Client) *DB { + return &DB{ + admin: &admin, + data: &data, + } +} + // WithInstance implements database.Driver func WithInstance(instance *DB, config *Config) (database.Driver, error) { if config == nil { @@ -72,6 +98,7 @@ func WithInstance(instance *DB, config *Config) (database.Driver, error) { sx := &Spanner{ db: instance, config: config, + lock: uatomic.NewUint32(unlockedVal), } if err := sx.ensureVersionTable(); err != nil { @@ -101,14 +128,21 @@ func (s *Spanner) Open(url string) (database.Driver, error) { } migrationsTable := purl.Query().Get("x-migrations-table") - if len(migrationsTable) == 0 { - migrationsTable = DefaultMigrationsTable + + cleanQuery := purl.Query().Get("x-clean-statements") + clean := false + if cleanQuery != "" { + clean, err = strconv.ParseBool(cleanQuery) + if err != nil { + return nil, err + } } db := &DB{admin: adminClient, data: dataClient} return WithInstance(db, &Config{ DatabaseName: dbname, MigrationsTable: migrationsTable, + CleanStatements: clean, }) } @@ -121,12 +155,18 @@ func (s *Spanner) Close() error { // Lock implements database.Driver but doesn't do anything because Spanner only // enqueues the UpdateDatabaseDdlRequest. func (s *Spanner) Lock() error { - return nil + if swapped := s.lock.CAS(unlockedVal, lockedVal); swapped { + return nil + } + return ErrLockHeld } // Unlock implements database.Driver but no action required, see Lock. func (s *Spanner) Unlock() error { - return nil + if swapped := s.lock.CAS(lockedVal, unlockedVal); swapped { + return nil + } + return ErrLockNotHeld } // Run implements database.Driver @@ -136,10 +176,15 @@ func (s *Spanner) Run(migration io.Reader) error { return err } - // run migration - stmts := migrationStatements(migr) - ctx := context.Background() + stmts := []string{string(migr)} + if s.config.CleanStatements { + stmts, err = cleanStatements(migr) + if err != nil { + return err + } + } + ctx := context.Background() op, err := s.db.admin.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{ Database: s.config.DatabaseName, Statements: stmts, @@ -204,6 +249,8 @@ func (s *Spanner) Version() (version int, dirty bool, err error) { return version, dirty, nil } +var nameMatcher = regexp.MustCompile(`(CREATE TABLE\s(\S+)\s)|(CREATE.+INDEX\s(\S+)\s)`) + // Drop implements database.Driver. Retrieves the database schema first and // creates statements to drop the indexes and tables accordingly. // Note: The drop statements are created in reverse order to how they're @@ -222,11 +269,10 @@ func (s *Spanner) Drop() error { return nil } - r := regexp.MustCompile(`(CREATE TABLE\s(\S+)\s)|(CREATE.+INDEX\s(\S+)\s)`) stmts := make([]string, 0) for i := len(res.Statements) - 1; i >= 0; i-- { s := res.Statements[i] - m := r.FindSubmatch([]byte(s)) + m := nameMatcher.FindSubmatch([]byte(s)) if len(m) == 0 { continue @@ -248,14 +294,27 @@ func (s *Spanner) Drop() error { return &database.Error{OrigErr: err, Query: []byte(strings.Join(stmts, "; "))} } - if err := s.ensureVersionTable(); err != nil { + return nil +} + +// ensureVersionTable checks if versions table exists and, if not, creates it. +// Note that this function locks the database, which deviates from the usual +// convention of "caller locks" in the Spanner type. +func (s *Spanner) ensureVersionTable() (err error) { + if err = s.Lock(); err != nil { return err } - return nil -} + defer func() { + if e := s.Unlock(); e != nil { + if err == nil { + err = e + } else { + err = multierror.Append(err, e) + } + } + }() -func (s *Spanner) ensureVersionTable() error { ctx := context.Background() tbl := s.config.MigrationsTable iter := s.db.data.Single().Read(ctx, tbl, spanner.AllKeys(), []string{"Version"}) @@ -283,12 +342,17 @@ func (s *Spanner) ensureVersionTable() error { return nil } -func migrationStatements(migration []byte) []string { - regex := regexp.MustCompile(";$") - migrationString := string(migration[:]) - migrationString = strings.TrimSpace(migrationString) - migrationString = regex.ReplaceAllString(migrationString, "") - - statements := strings.Split(migrationString, ";") - return statements +func cleanStatements(migration []byte) ([]string, error) { + // The Spanner GCP backend does not yet support comments for the UpdateDatabaseDdl RPC + // (see https://issuetracker.google.com/issues/159730604) we use + // spansql to parse the DDL and output valid stamements without comments + ddl, err := spansql.ParseDDL("", string(migration)) + if err != nil { + return nil, err + } + stmts := make([]string, 0, len(ddl.List)) + for _, stmt := range ddl.List { + stmts = append(stmts, stmt.SQL()) + } + return stmts, nil } diff --git a/database/spanner/spanner_test.go b/database/spanner/spanner_test.go index bd90d6530..d6ab4db32 100644 --- a/database/spanner/spanner_test.go +++ b/database/spanner/spanner_test.go @@ -5,24 +5,167 @@ import ( "os" "testing" + "github.com/golang-migrate/migrate/v4" + dt "github.com/golang-migrate/migrate/v4/database/testing" + _ "github.com/golang-migrate/migrate/v4/source/file" + + "cloud.google.com/go/spanner/spannertest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func Test(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") +// withSpannerEmulator is not thread-safe and cannot be used with parallel tests since it sets the emulator +func withSpannerEmulator(t *testing.T, testFunc func(t *testing.T)) { + t.Helper() + srv, err := spannertest.NewServer("localhost:0") + if err != nil { + t.Fatal("Failed to create Spanner emulator:", err) + } + // This is not thread-safe + if err := os.Setenv("SPANNER_EMULATOR_HOST", srv.Addr); err != nil { + t.Fatal("Failed to set SPANNER_EMULATOR_HOST env var:", err) } + defer srv.Close() + testFunc(t) + +} + +const db = "projects/abc/instances/def/databases/testdb" + +func Test(t *testing.T) { + withSpannerEmulator(t, func(t *testing.T) { + uri := fmt.Sprintf("spanner://%s", db) + s := &Spanner{} + d, err := s.Open(uri) + if err != nil { + t.Fatal(err) + } + dt.Test(t, d, []byte("CREATE TABLE test (id BOOL) PRIMARY KEY (id)")) + }) +} + +func TestMigrate(t *testing.T) { + withSpannerEmulator(t, func(t *testing.T) { + s := &Spanner{} + uri := fmt.Sprintf("spanner://%s", db) + d, err := s.Open(uri) + if err != nil { + t.Fatal(err) + } + m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", uri, d) + if err != nil { + t.Fatal(err) + } + dt.TestMigrate(t, m) + }) +} + +func TestCleanStatements(t *testing.T) { + testCases := []struct { + name string + multiStatement string + expected []string + }{ + { + name: "no statement", + multiStatement: "", + expected: []string{}, + }, + { + name: "single statement, single line, no semicolon, no comment", + multiStatement: "CREATE TABLE table_name (id STRING(255) NOT NULL) PRIMARY KEY (id)", + expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n) PRIMARY KEY(id)"}, + }, + { + name: "single statement, multi line, no semicolon, no comment", + multiStatement: `CREATE TABLE table_name ( + id STRING(255) NOT NULL, + ) PRIMARY KEY (id)`, + expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n) PRIMARY KEY(id)"}, + }, + { + name: "single statement, single line, with semicolon, no comment", + multiStatement: "CREATE TABLE table_name (id STRING(255) NOT NULL) PRIMARY KEY (id);", + expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n) PRIMARY KEY(id)"}, + }, + { + name: "single statement, multi line, with semicolon, no comment", + multiStatement: `CREATE TABLE table_name ( + id STRING(255) NOT NULL, + ) PRIMARY KEY (id);`, + expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n) PRIMARY KEY(id)"}, + }, + { + name: "multi statement, with trailing semicolon. no comment", + // From https://github.com/mattes/migrate/pull/281 + multiStatement: `CREATE TABLE table_name ( + id STRING(255) NOT NULL, + ) PRIMARY KEY(id); - db, ok := os.LookupEnv("SPANNER_DATABASE") - if !ok { - t.Skip("SPANNER_DATABASE not set, skipping test.") + CREATE INDEX table_name_id_idx ON table_name (id);`, + expected: []string{`CREATE TABLE table_name ( + id STRING(255) NOT NULL, +) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"}, + }, + { + name: "multi statement, no trailing semicolon, no comment", + // From https://github.com/mattes/migrate/pull/281 + multiStatement: `CREATE TABLE table_name ( + id STRING(255) NOT NULL, + ) PRIMARY KEY(id); + + CREATE INDEX table_name_id_idx ON table_name (id)`, + expected: []string{`CREATE TABLE table_name ( + id STRING(255) NOT NULL, +) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"}, + }, + { + name: "multi statement, no trailing semicolon, standalone comment", + // From https://github.com/mattes/migrate/pull/281 + multiStatement: `CREATE TABLE table_name ( + -- standalone comment + id STRING(255) NOT NULL, + ) PRIMARY KEY(id); + + CREATE INDEX table_name_id_idx ON table_name (id)`, + expected: []string{`CREATE TABLE table_name ( + id STRING(255) NOT NULL, +) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"}, + }, + { + name: "multi statement, no trailing semicolon, inline comment", + // From https://github.com/mattes/migrate/pull/281 + multiStatement: `CREATE TABLE table_name ( + id STRING(255) NOT NULL, -- inline comment + ) PRIMARY KEY(id); + + CREATE INDEX table_name_id_idx ON table_name (id)`, + expected: []string{`CREATE TABLE table_name ( + id STRING(255) NOT NULL, +) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"}, + }, + { + name: "alter table with SET OPTIONS", + multiStatement: `ALTER TABLE users ALTER COLUMN created + SET OPTIONS (allow_commit_timestamp=true);`, + expected: []string{"ALTER TABLE users ALTER COLUMN created SET OPTIONS (allow_commit_timestamp = true)"}, + }, + { + name: "column with NUMERIC type", + multiStatement: `CREATE TABLE table_name ( + id STRING(255) NOT NULL, + sum NUMERIC, + ) PRIMARY KEY (id)`, + expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n sum NUMERIC,\n) PRIMARY KEY(id)"}, + }, } - s := &Spanner{} - addr := fmt.Sprintf("spanner://%v", db) - d, err := s.Open(addr) - if err != nil { - t.Fatalf("%v", err) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + stmts, err := cleanStatements([]byte(tc.multiStatement)) + require.NoError(t, err, "Error cleaning statements") + assert.Equal(t, tc.expected, stmts) + }) } - dt.Test(t, d, []byte("SELECT 1")) } diff --git a/database/sqlcipher/README.md b/database/sqlcipher/README.md new file mode 100644 index 000000000..5fda5b361 --- /dev/null +++ b/database/sqlcipher/README.md @@ -0,0 +1,3 @@ +# sqlcipher + +This is just a copy of the [sqlite3](https://github.com/golang-migrate/migrate/blob/master/database/sqlite3) driver except that it imports `github.com/mutecomm/go-sqlcipher`. \ No newline at end of file diff --git a/database/sqlite3/migration/33_create_table.down.sql b/database/sqlcipher/examples/migrations/33_create_table.down.sql similarity index 100% rename from database/sqlite3/migration/33_create_table.down.sql rename to database/sqlcipher/examples/migrations/33_create_table.down.sql diff --git a/database/sqlite3/migration/33_create_table.up.sql b/database/sqlcipher/examples/migrations/33_create_table.up.sql similarity index 100% rename from database/sqlite3/migration/33_create_table.up.sql rename to database/sqlcipher/examples/migrations/33_create_table.up.sql diff --git a/database/sqlite3/migration/44_alter_table.down.sql b/database/sqlcipher/examples/migrations/44_alter_table.down.sql similarity index 100% rename from database/sqlite3/migration/44_alter_table.down.sql rename to database/sqlcipher/examples/migrations/44_alter_table.down.sql diff --git a/database/sqlite3/migration/44_alter_table.up.sql b/database/sqlcipher/examples/migrations/44_alter_table.up.sql similarity index 100% rename from database/sqlite3/migration/44_alter_table.up.sql rename to database/sqlcipher/examples/migrations/44_alter_table.up.sql diff --git a/database/sqlcipher/sqlcipher.go b/database/sqlcipher/sqlcipher.go new file mode 100644 index 000000000..782eed24b --- /dev/null +++ b/database/sqlcipher/sqlcipher.go @@ -0,0 +1,269 @@ +package sqlcipher + +import ( + "database/sql" + "fmt" + "go.uber.org/atomic" + "io" + "io/ioutil" + nurl "net/url" + "strconv" + "strings" + + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database" + "github.com/hashicorp/go-multierror" + _ "github.com/mutecomm/go-sqlcipher/v4" +) + +func init() { + database.Register("sqlcipher", &Sqlite{}) +} + +var DefaultMigrationsTable = "schema_migrations" +var ( + ErrDatabaseDirty = fmt.Errorf("database is dirty") + ErrNilConfig = fmt.Errorf("no config") + ErrNoDatabaseName = fmt.Errorf("no database name") +) + +type Config struct { + MigrationsTable string + DatabaseName string + NoTxWrap bool +} + +type Sqlite struct { + db *sql.DB + isLocked atomic.Bool + + config *Config +} + +func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { + if config == nil { + return nil, ErrNilConfig + } + + if err := instance.Ping(); err != nil { + return nil, err + } + + if len(config.MigrationsTable) == 0 { + config.MigrationsTable = DefaultMigrationsTable + } + + mx := &Sqlite{ + db: instance, + config: config, + } + if err := mx.ensureVersionTable(); err != nil { + return nil, err + } + return mx, nil +} + +// ensureVersionTable checks if versions table exists and, if not, creates it. +// Note that this function locks the database, which deviates from the usual +// convention of "caller locks" in the Sqlite type. +func (m *Sqlite) ensureVersionTable() (err error) { + if err = m.Lock(); err != nil { + return err + } + + defer func() { + if e := m.Unlock(); e != nil { + if err == nil { + err = e + } else { + err = multierror.Append(err, e) + } + } + }() + + query := fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s (version uint64,dirty bool); + CREATE UNIQUE INDEX IF NOT EXISTS version_unique ON %s (version); + `, m.config.MigrationsTable, m.config.MigrationsTable) + + if _, err := m.db.Exec(query); err != nil { + return err + } + return nil +} + +func (m *Sqlite) Open(url string) (database.Driver, error) { + purl, err := nurl.Parse(url) + if err != nil { + return nil, err + } + dbfile := strings.Replace(migrate.FilterCustomQuery(purl).String(), "sqlite3://", "", 1) + db, err := sql.Open("sqlite3", dbfile) + if err != nil { + return nil, err + } + + qv := purl.Query() + + migrationsTable := qv.Get("x-migrations-table") + if len(migrationsTable) == 0 { + migrationsTable = DefaultMigrationsTable + } + + noTxWrap := false + if v := qv.Get("x-no-tx-wrap"); v != "" { + noTxWrap, err = strconv.ParseBool(v) + if err != nil { + return nil, fmt.Errorf("x-no-tx-wrap: %s", err) + } + } + + mx, err := WithInstance(db, &Config{ + DatabaseName: purl.Path, + MigrationsTable: migrationsTable, + NoTxWrap: noTxWrap, + }) + if err != nil { + return nil, err + } + return mx, nil +} + +func (m *Sqlite) Close() error { + return m.db.Close() +} + +func (m *Sqlite) Drop() (err error) { + query := `SELECT name FROM sqlite_master WHERE type = 'table';` + tables, err := m.db.Query(query) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + defer func() { + if errClose := tables.Close(); errClose != nil { + err = multierror.Append(err, errClose) + } + }() + + tableNames := make([]string, 0) + for tables.Next() { + var tableName string + if err := tables.Scan(&tableName); err != nil { + return err + } + if len(tableName) > 0 { + tableNames = append(tableNames, tableName) + } + } + if err := tables.Err(); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + if len(tableNames) > 0 { + for _, t := range tableNames { + query := "DROP TABLE " + t + err = m.executeQuery(query) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + query := "VACUUM" + _, err = m.db.Query(query) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + + return nil +} + +func (m *Sqlite) Lock() error { + if !m.isLocked.CAS(false, true) { + return database.ErrLocked + } + return nil +} + +func (m *Sqlite) Unlock() error { + if !m.isLocked.CAS(true, false) { + return database.ErrNotLocked + } + return nil +} + +func (m *Sqlite) Run(migration io.Reader) error { + migr, err := ioutil.ReadAll(migration) + if err != nil { + return err + } + query := string(migr[:]) + + if m.config.NoTxWrap { + return m.executeQueryNoTx(query) + } + return m.executeQuery(query) +} + +func (m *Sqlite) executeQuery(query string) error { + tx, err := m.db.Begin() + if err != nil { + return &database.Error{OrigErr: err, Err: "transaction start failed"} + } + if _, err := tx.Exec(query); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } + return &database.Error{OrigErr: err, Query: []byte(query)} + } + if err := tx.Commit(); err != nil { + return &database.Error{OrigErr: err, Err: "transaction commit failed"} + } + return nil +} + +func (m *Sqlite) executeQueryNoTx(query string) error { + if _, err := m.db.Exec(query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + return nil +} + +func (m *Sqlite) SetVersion(version int, dirty bool) error { + tx, err := m.db.Begin() + if err != nil { + return &database.Error{OrigErr: err, Err: "transaction start failed"} + } + + query := "DELETE FROM " + m.config.MigrationsTable + if _, err := tx.Exec(query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + // Also re-write the schema version for nil dirty versions to prevent + // empty schema version for failed down migration on the first migration + // See: https://github.com/golang-migrate/migrate/issues/330 + if version >= 0 || (version == database.NilVersion && dirty) { + query := fmt.Sprintf(`INSERT INTO %s (version, dirty) VALUES (?, ?)`, m.config.MigrationsTable) + if _, err := tx.Exec(query, version, dirty); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + + if err := tx.Commit(); err != nil { + return &database.Error{OrigErr: err, Err: "transaction commit failed"} + } + + return nil +} + +func (m *Sqlite) Version() (version int, dirty bool, err error) { + query := "SELECT version, dirty FROM " + m.config.MigrationsTable + " LIMIT 1" + err = m.db.QueryRow(query).Scan(&version, &dirty) + if err != nil { + return database.NilVersion, false, nil + } + return version, dirty, nil +} diff --git a/database/sqlcipher/sqlcipher_test.go b/database/sqlcipher/sqlcipher_test.go new file mode 100644 index 000000000..e75422316 --- /dev/null +++ b/database/sqlcipher/sqlcipher_test.go @@ -0,0 +1,162 @@ +package sqlcipher + +import ( + "database/sql" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/golang-migrate/migrate/v4" + dt "github.com/golang-migrate/migrate/v4/database/testing" + _ "github.com/golang-migrate/migrate/v4/source/file" + _ "github.com/mutecomm/go-sqlcipher/v4" +) + +func Test(t *testing.T) { + dir, err := ioutil.TempDir("", "sqlite3-driver-test") + if err != nil { + return + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + }() + t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite3.db")) + p := &Sqlite{} + addr := fmt.Sprintf("sqlite3://%s", filepath.Join(dir, "sqlite3.db")) + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + dt.Test(t, d, []byte("CREATE TABLE t (Qty int, Name string);")) +} + +func TestMigrate(t *testing.T) { + dir, err := ioutil.TempDir("", "sqlite3-driver-test") + if err != nil { + return + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + }() + t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite3.db")) + + db, err := sql.Open("sqlite3", filepath.Join(dir, "sqlite3.db")) + if err != nil { + return + } + defer func() { + if err := db.Close(); err != nil { + return + } + }() + driver, err := WithInstance(db, &Config{}) + if err != nil { + t.Fatal(err) + } + + m, err := migrate.NewWithDatabaseInstance( + "file://./examples/migrations", + "ql", driver) + if err != nil { + t.Fatal(err) + } + dt.TestMigrate(t, m) +} + +func TestMigrationTable(t *testing.T) { + dir, err := ioutil.TempDir("", "sqlite3-driver-test-migration-table") + if err != nil { + return + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + }() + + t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite3.db")) + + db, err := sql.Open("sqlite3", filepath.Join(dir, "sqlite3.db")) + if err != nil { + return + } + defer func() { + if err := db.Close(); err != nil { + return + } + }() + + config := &Config{ + MigrationsTable: "my_migration_table", + } + driver, err := WithInstance(db, config) + if err != nil { + t.Fatal(err) + } + m, err := migrate.NewWithDatabaseInstance( + "file://./examples/migrations", + "ql", driver) + if err != nil { + t.Fatal(err) + } + t.Log("UP") + err = m.Up() + if err != nil { + t.Fatal(err) + } + + _, err = db.Query(fmt.Sprintf("SELECT * FROM %s", config.MigrationsTable)) + if err != nil { + t.Fatal(err) + } +} + +func TestNoTxWrap(t *testing.T) { + dir, err := ioutil.TempDir("", "sqlite3-driver-test") + if err != nil { + return + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + }() + t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite3.db")) + p := &Sqlite{} + addr := fmt.Sprintf("sqlite3://%s?x-no-tx-wrap=true", filepath.Join(dir, "sqlite3.db")) + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + // An explicit BEGIN statement would ordinarily fail without x-no-tx-wrap. + // (Transactions in sqlite may not be nested.) + dt.Test(t, d, []byte("BEGIN; CREATE TABLE t (Qty int, Name string); COMMIT;")) +} + +func TestNoTxWrapInvalidValue(t *testing.T) { + dir, err := ioutil.TempDir("", "sqlite3-driver-test") + if err != nil { + return + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + }() + t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite3.db")) + p := &Sqlite{} + addr := fmt.Sprintf("sqlite3://%s?x-no-tx-wrap=yeppers", filepath.Join(dir, "sqlite3.db")) + _, err = p.Open(addr) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "x-no-tx-wrap") + assert.Contains(t, err.Error(), "invalid syntax") + } +} diff --git a/database/sqlite/README.md b/database/sqlite/README.md new file mode 100644 index 000000000..4cd018f32 --- /dev/null +++ b/database/sqlite/README.md @@ -0,0 +1,17 @@ +# sqlite + +`sqlite://path/to/database?query` + +Unlike other migrate database drivers, the sqlite driver will automatically wrap each migration in an implicit transaction by default. Migrations must not contain explicit `BEGIN` or `COMMIT` statements. This behavior may change in a future major release. (See below for a workaround.) + +The auxiliary query parameters listed below may be supplied to tailor migrate behavior. All auxiliary query parameters are optional. + +| URL Query | WithInstance Config | Description | +|------------|---------------------|-------------| +| `x-migrations-table` | `MigrationsTable` | Name of the migrations table. Defaults to `schema_migrations`. | +| `x-no-tx-wrap` | `NoTxWrap` | Disable implicit transactions when `true`. Migrations may, and should, contain explicit `BEGIN` and `COMMIT` statements. | + +## Notes + +* Uses the `modernc.org/sqlite` sqlite db driver (pure Go) + * Has [limited `GOOS` and `GOARCH` support](https://pkg.go.dev/modernc.org/sqlite?utm_source=godoc#hdr-Supported_platforms_and_architectures) diff --git a/database/sqlite/examples/migrations/33_create_table.down.sql b/database/sqlite/examples/migrations/33_create_table.down.sql new file mode 100644 index 000000000..72d18c554 --- /dev/null +++ b/database/sqlite/examples/migrations/33_create_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS pets; \ No newline at end of file diff --git a/database/sqlite/examples/migrations/33_create_table.up.sql b/database/sqlite/examples/migrations/33_create_table.up.sql new file mode 100644 index 000000000..5ad3404d1 --- /dev/null +++ b/database/sqlite/examples/migrations/33_create_table.up.sql @@ -0,0 +1,3 @@ +CREATE TABLE pets ( + name string +); \ No newline at end of file diff --git a/database/sqlite/examples/migrations/44_alter_table.down.sql b/database/sqlite/examples/migrations/44_alter_table.down.sql new file mode 100644 index 000000000..72d18c554 --- /dev/null +++ b/database/sqlite/examples/migrations/44_alter_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS pets; \ No newline at end of file diff --git a/database/sqlite/examples/migrations/44_alter_table.up.sql b/database/sqlite/examples/migrations/44_alter_table.up.sql new file mode 100644 index 000000000..f0682fcca --- /dev/null +++ b/database/sqlite/examples/migrations/44_alter_table.up.sql @@ -0,0 +1 @@ +ALTER TABLE pets ADD predator bool; diff --git a/database/sqlite/sqlite.go b/database/sqlite/sqlite.go new file mode 100644 index 000000000..d33c60e46 --- /dev/null +++ b/database/sqlite/sqlite.go @@ -0,0 +1,269 @@ +package sqlite + +import ( + "database/sql" + "fmt" + "go.uber.org/atomic" + "io" + "io/ioutil" + nurl "net/url" + "strconv" + "strings" + + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database" + "github.com/hashicorp/go-multierror" + _ "modernc.org/sqlite" +) + +func init() { + database.Register("sqlite", &Sqlite{}) +} + +var DefaultMigrationsTable = "schema_migrations" +var ( + ErrDatabaseDirty = fmt.Errorf("database is dirty") + ErrNilConfig = fmt.Errorf("no config") + ErrNoDatabaseName = fmt.Errorf("no database name") +) + +type Config struct { + MigrationsTable string + DatabaseName string + NoTxWrap bool +} + +type Sqlite struct { + db *sql.DB + isLocked atomic.Bool + + config *Config +} + +func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { + if config == nil { + return nil, ErrNilConfig + } + + if err := instance.Ping(); err != nil { + return nil, err + } + + if len(config.MigrationsTable) == 0 { + config.MigrationsTable = DefaultMigrationsTable + } + + mx := &Sqlite{ + db: instance, + config: config, + } + if err := mx.ensureVersionTable(); err != nil { + return nil, err + } + return mx, nil +} + +// ensureVersionTable checks if versions table exists and, if not, creates it. +// Note that this function locks the database, which deviates from the usual +// convention of "caller locks" in the Sqlite type. +func (m *Sqlite) ensureVersionTable() (err error) { + if err = m.Lock(); err != nil { + return err + } + + defer func() { + if e := m.Unlock(); e != nil { + if err == nil { + err = e + } else { + err = multierror.Append(err, e) + } + } + }() + + query := fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s (version uint64,dirty bool); + CREATE UNIQUE INDEX IF NOT EXISTS version_unique ON %s (version); + `, m.config.MigrationsTable, m.config.MigrationsTable) + + if _, err := m.db.Exec(query); err != nil { + return err + } + return nil +} + +func (m *Sqlite) Open(url string) (database.Driver, error) { + purl, err := nurl.Parse(url) + if err != nil { + return nil, err + } + dbfile := strings.Replace(migrate.FilterCustomQuery(purl).String(), "sqlite://", "", 1) + db, err := sql.Open("sqlite", dbfile) + if err != nil { + return nil, err + } + + qv := purl.Query() + + migrationsTable := qv.Get("x-migrations-table") + if len(migrationsTable) == 0 { + migrationsTable = DefaultMigrationsTable + } + + noTxWrap := false + if v := qv.Get("x-no-tx-wrap"); v != "" { + noTxWrap, err = strconv.ParseBool(v) + if err != nil { + return nil, fmt.Errorf("x-no-tx-wrap: %s", err) + } + } + + mx, err := WithInstance(db, &Config{ + DatabaseName: purl.Path, + MigrationsTable: migrationsTable, + NoTxWrap: noTxWrap, + }) + if err != nil { + return nil, err + } + return mx, nil +} + +func (m *Sqlite) Close() error { + return m.db.Close() +} + +func (m *Sqlite) Drop() (err error) { + query := `SELECT name FROM sqlite_master WHERE type = 'table';` + tables, err := m.db.Query(query) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + defer func() { + if errClose := tables.Close(); errClose != nil { + err = multierror.Append(err, errClose) + } + }() + + tableNames := make([]string, 0) + for tables.Next() { + var tableName string + if err := tables.Scan(&tableName); err != nil { + return err + } + if len(tableName) > 0 { + tableNames = append(tableNames, tableName) + } + } + if err := tables.Err(); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + if len(tableNames) > 0 { + for _, t := range tableNames { + query := "DROP TABLE " + t + err = m.executeQuery(query) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + query := "VACUUM" + _, err = m.db.Query(query) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + + return nil +} + +func (m *Sqlite) Lock() error { + if !m.isLocked.CAS(false, true) { + return database.ErrLocked + } + return nil +} + +func (m *Sqlite) Unlock() error { + if !m.isLocked.CAS(true, false) { + return database.ErrNotLocked + } + return nil +} + +func (m *Sqlite) Run(migration io.Reader) error { + migr, err := ioutil.ReadAll(migration) + if err != nil { + return err + } + query := string(migr[:]) + + if m.config.NoTxWrap { + return m.executeQueryNoTx(query) + } + return m.executeQuery(query) +} + +func (m *Sqlite) executeQuery(query string) error { + tx, err := m.db.Begin() + if err != nil { + return &database.Error{OrigErr: err, Err: "transaction start failed"} + } + if _, err := tx.Exec(query); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } + return &database.Error{OrigErr: err, Query: []byte(query)} + } + if err := tx.Commit(); err != nil { + return &database.Error{OrigErr: err, Err: "transaction commit failed"} + } + return nil +} + +func (m *Sqlite) executeQueryNoTx(query string) error { + if _, err := m.db.Exec(query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + return nil +} + +func (m *Sqlite) SetVersion(version int, dirty bool) error { + tx, err := m.db.Begin() + if err != nil { + return &database.Error{OrigErr: err, Err: "transaction start failed"} + } + + query := "DELETE FROM " + m.config.MigrationsTable + if _, err := tx.Exec(query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + // Also re-write the schema version for nil dirty versions to prevent + // empty schema version for failed down migration on the first migration + // See: https://github.com/golang-migrate/migrate/issues/330 + if version >= 0 || (version == database.NilVersion && dirty) { + query := fmt.Sprintf(`INSERT INTO %s (version, dirty) VALUES (?, ?)`, m.config.MigrationsTable) + if _, err := tx.Exec(query, version, dirty); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + + if err := tx.Commit(); err != nil { + return &database.Error{OrigErr: err, Err: "transaction commit failed"} + } + + return nil +} + +func (m *Sqlite) Version() (version int, dirty bool, err error) { + query := "SELECT version, dirty FROM " + m.config.MigrationsTable + " LIMIT 1" + err = m.db.QueryRow(query).Scan(&version, &dirty) + if err != nil { + return database.NilVersion, false, nil + } + return version, dirty, nil +} diff --git a/database/sqlite/sqlite_test.go b/database/sqlite/sqlite_test.go new file mode 100644 index 000000000..8db113115 --- /dev/null +++ b/database/sqlite/sqlite_test.go @@ -0,0 +1,183 @@ +package sqlite + +import ( + "database/sql" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/golang-migrate/migrate/v4" + dt "github.com/golang-migrate/migrate/v4/database/testing" + _ "github.com/golang-migrate/migrate/v4/source/file" + _ "modernc.org/sqlite" +) + +func Test(t *testing.T) { + dir, err := ioutil.TempDir("", "sqlite-driver-test") + if err != nil { + return + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + }() + t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite.db")) + p := &Sqlite{} + addr := fmt.Sprintf("sqlite://%s", filepath.Join(dir, "sqlite.db")) + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + dt.Test(t, d, []byte("CREATE TABLE t (Qty int, Name string);")) +} + +func TestMigrate(t *testing.T) { + dir, err := ioutil.TempDir("", "sqlite-driver-test") + if err != nil { + return + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + }() + t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite.db")) + + db, err := sql.Open("sqlite", filepath.Join(dir, "sqlite.db")) + if err != nil { + return + } + defer func() { + if err := db.Close(); err != nil { + return + } + }() + driver, err := WithInstance(db, &Config{}) + if err != nil { + t.Fatal(err) + } + + m, err := migrate.NewWithDatabaseInstance( + "file://./examples/migrations", + "ql", driver) + if err != nil { + t.Fatal(err) + } + dt.TestMigrate(t, m) +} + +func TestMigrationTable(t *testing.T) { + dir, err := ioutil.TempDir("", "sqlite-driver-test-migration-table") + if err != nil { + return + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + }() + + t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite.db")) + + db, err := sql.Open("sqlite", filepath.Join(dir, "sqlite.db")) + if err != nil { + return + } + defer func() { + if err := db.Close(); err != nil { + return + } + }() + + config := &Config{ + MigrationsTable: "my_migration_table", + } + driver, err := WithInstance(db, config) + if err != nil { + t.Fatal(err) + } + m, err := migrate.NewWithDatabaseInstance( + "file://./examples/migrations", + "ql", driver) + if err != nil { + t.Fatal(err) + } + t.Log("UP") + err = m.Up() + if err != nil { + t.Fatal(err) + } + + _, err = db.Query(fmt.Sprintf("SELECT * FROM %s", config.MigrationsTable)) + if err != nil { + t.Fatal(err) + } +} + +func TestNoTxWrap(t *testing.T) { + dir, err := ioutil.TempDir("", "sqlite-driver-test") + if err != nil { + return + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + }() + t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite.db")) + p := &Sqlite{} + addr := fmt.Sprintf("sqlite://%s?x-no-tx-wrap=true", filepath.Join(dir, "sqlite.db")) + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + // An explicit BEGIN statement would ordinarily fail without x-no-tx-wrap. + // (Transactions in sqlite may not be nested.) + dt.Test(t, d, []byte("BEGIN; CREATE TABLE t (Qty int, Name string); COMMIT;")) +} + +func TestNoTxWrapInvalidValue(t *testing.T) { + dir, err := ioutil.TempDir("", "sqlite-driver-test") + if err != nil { + return + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + }() + t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite.db")) + p := &Sqlite{} + addr := fmt.Sprintf("sqlite://%s?x-no-tx-wrap=yeppers", filepath.Join(dir, "sqlite.db")) + _, err = p.Open(addr) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "x-no-tx-wrap") + assert.Contains(t, err.Error(), "invalid syntax") + } +} + +func TestMigrateWithDirectoryNameContainsWhitespaces(t *testing.T) { + dir, err := ioutil.TempDir("", "directory name contains whitespaces") + if err != nil { + return + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + }() + dbPath := filepath.Join(dir, "sqlite.db") + t.Logf("DB path : %s\n", dbPath) + p := &Sqlite{} + addr := fmt.Sprintf("sqlite://file:%s", dbPath) + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + dt.Test(t, d, []byte("CREATE TABLE t (Qty int, Name string);")) +} diff --git a/database/sqlite3/README.md b/database/sqlite3/README.md index e69de29bb..13f3f0d14 100644 --- a/database/sqlite3/README.md +++ b/database/sqlite3/README.md @@ -0,0 +1,16 @@ +# sqlite3 + +`sqlite3://path/to/database?query` + +Unlike other migrate database drivers, the sqlite3 driver will automatically wrap each migration in an implicit transaction by default. Migrations must not contain explicit `BEGIN` or `COMMIT` statements. This behavior may change in a future major release. (See below for a workaround.) + +Refer to [upstream documentation](https://github.com/mattn/go-sqlite3/blob/master/README.md#connection-string) for a complete list of query parameters supported by the sqlite3 database driver. The auxiliary query parameters listed below may be supplied to tailor migrate behavior. All auxiliary query parameters are optional. + +| URL Query | WithInstance Config | Description | +|------------|---------------------|-------------| +| `x-migrations-table` | `MigrationsTable` | Name of the migrations table. Defaults to `schema_migrations`. | +| `x-no-tx-wrap` | `NoTxWrap` | Disable implicit transactions when `true`. Migrations may, and should, contain explicit `BEGIN` and `COMMIT` statements. | + +## Notes + +* Uses the `github.com/mattn/go-sqlite3` sqlite db driver (cgo) diff --git a/database/sqlite3/examples/migrations/33_create_table.down.sql b/database/sqlite3/examples/migrations/33_create_table.down.sql new file mode 100644 index 000000000..72d18c554 --- /dev/null +++ b/database/sqlite3/examples/migrations/33_create_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS pets; \ No newline at end of file diff --git a/database/sqlite3/examples/migrations/33_create_table.up.sql b/database/sqlite3/examples/migrations/33_create_table.up.sql new file mode 100644 index 000000000..5ad3404d1 --- /dev/null +++ b/database/sqlite3/examples/migrations/33_create_table.up.sql @@ -0,0 +1,3 @@ +CREATE TABLE pets ( + name string +); \ No newline at end of file diff --git a/database/sqlite3/examples/migrations/44_alter_table.down.sql b/database/sqlite3/examples/migrations/44_alter_table.down.sql new file mode 100644 index 000000000..72d18c554 --- /dev/null +++ b/database/sqlite3/examples/migrations/44_alter_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS pets; \ No newline at end of file diff --git a/database/sqlite3/examples/migrations/44_alter_table.up.sql b/database/sqlite3/examples/migrations/44_alter_table.up.sql new file mode 100644 index 000000000..f0682fcca --- /dev/null +++ b/database/sqlite3/examples/migrations/44_alter_table.up.sql @@ -0,0 +1 @@ +ALTER TABLE pets ADD predator bool; diff --git a/database/sqlite3/sqlite3.go b/database/sqlite3/sqlite3.go index 176ee0d43..65aa6e74c 100644 --- a/database/sqlite3/sqlite3.go +++ b/database/sqlite3/sqlite3.go @@ -3,13 +3,17 @@ package sqlite3 import ( "database/sql" "fmt" - "github.com/golang-migrate/migrate/v4" - "github.com/golang-migrate/migrate/v4/database" - _ "github.com/mattn/go-sqlite3" + "go.uber.org/atomic" "io" "io/ioutil" nurl "net/url" + "strconv" "strings" + + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database" + "github.com/hashicorp/go-multierror" + _ "github.com/mattn/go-sqlite3" ) func init() { @@ -26,11 +30,12 @@ var ( type Config struct { MigrationsTable string DatabaseName string + NoTxWrap bool } type Sqlite struct { db *sql.DB - isLocked bool + isLocked atomic.Bool config *Config } @@ -43,6 +48,7 @@ func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { if err := instance.Ping(); err != nil { return nil, err } + if len(config.MigrationsTable) == 0 { config.MigrationsTable = DefaultMigrationsTable } @@ -57,12 +63,28 @@ func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { return mx, nil } -func (m *Sqlite) ensureVersionTable() error { +// ensureVersionTable checks if versions table exists and, if not, creates it. +// Note that this function locks the database, which deviates from the usual +// convention of "caller locks" in the Sqlite type. +func (m *Sqlite) ensureVersionTable() (err error) { + if err = m.Lock(); err != nil { + return err + } + + defer func() { + if e := m.Unlock(); e != nil { + if err == nil { + err = e + } else { + err = multierror.Append(err, e) + } + } + }() query := fmt.Sprintf(` CREATE TABLE IF NOT EXISTS %s (version uint64,dirty bool); CREATE UNIQUE INDEX IF NOT EXISTS version_unique ON %s (version); - `, DefaultMigrationsTable, DefaultMigrationsTable) + `, m.config.MigrationsTable, m.config.MigrationsTable) if _, err := m.db.Exec(query); err != nil { return err @@ -81,13 +103,25 @@ func (m *Sqlite) Open(url string) (database.Driver, error) { return nil, err } - migrationsTable := purl.Query().Get("x-migrations-table") + qv := purl.Query() + + migrationsTable := qv.Get("x-migrations-table") if len(migrationsTable) == 0 { migrationsTable = DefaultMigrationsTable } + + noTxWrap := false + if v := qv.Get("x-no-tx-wrap"); v != "" { + noTxWrap, err = strconv.ParseBool(v) + if err != nil { + return nil, fmt.Errorf("x-no-tx-wrap: %s", err) + } + } + mx, err := WithInstance(db, &Config{ DatabaseName: purl.Path, MigrationsTable: migrationsTable, + NoTxWrap: noTxWrap, }) if err != nil { return nil, err @@ -99,13 +133,18 @@ func (m *Sqlite) Close() error { return m.db.Close() } -func (m *Sqlite) Drop() error { +func (m *Sqlite) Drop() (err error) { query := `SELECT name FROM sqlite_master WHERE type = 'table';` tables, err := m.db.Query(query) if err != nil { return &database.Error{OrigErr: err, Query: []byte(query)} } - defer tables.Close() + defer func() { + if errClose := tables.Close(); errClose != nil { + err = multierror.Append(err, errClose) + } + }() + tableNames := make([]string, 0) for tables.Next() { var tableName string @@ -116,6 +155,10 @@ func (m *Sqlite) Drop() error { tableNames = append(tableNames, tableName) } } + if err := tables.Err(); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + if len(tableNames) > 0 { for _, t := range tableNames { query := "DROP TABLE " + t @@ -124,9 +167,6 @@ func (m *Sqlite) Drop() error { return &database.Error{OrigErr: err, Query: []byte(query)} } } - if err := m.ensureVersionTable(); err != nil { - return err - } query := "VACUUM" _, err = m.db.Query(query) if err != nil { @@ -138,18 +178,16 @@ func (m *Sqlite) Drop() error { } func (m *Sqlite) Lock() error { - if m.isLocked { + if !m.isLocked.CAS(false, true) { return database.ErrLocked } - m.isLocked = true return nil } func (m *Sqlite) Unlock() error { - if !m.isLocked { - return nil + if !m.isLocked.CAS(true, false) { + return database.ErrNotLocked } - m.isLocked = false return nil } @@ -160,6 +198,9 @@ func (m *Sqlite) Run(migration io.Reader) error { } query := string(migr[:]) + if m.config.NoTxWrap { + return m.executeQueryNoTx(query) + } return m.executeQuery(query) } @@ -169,7 +210,9 @@ func (m *Sqlite) executeQuery(query string) error { return &database.Error{OrigErr: err, Err: "transaction start failed"} } if _, err := tx.Exec(query); err != nil { - tx.Rollback() + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } return &database.Error{OrigErr: err, Query: []byte(query)} } if err := tx.Commit(); err != nil { @@ -178,6 +221,13 @@ func (m *Sqlite) executeQuery(query string) error { return nil } +func (m *Sqlite) executeQueryNoTx(query string) error { + if _, err := m.db.Exec(query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + return nil +} + func (m *Sqlite) SetVersion(version int, dirty bool) error { tx, err := m.db.Begin() if err != nil { @@ -189,10 +239,15 @@ func (m *Sqlite) SetVersion(version int, dirty bool) error { return &database.Error{OrigErr: err, Query: []byte(query)} } - if version >= 0 { - query := fmt.Sprintf(`INSERT INTO %s (version, dirty) VALUES (%d, '%t')`, m.config.MigrationsTable, version, dirty) - if _, err := tx.Exec(query); err != nil { - tx.Rollback() + // Also re-write the schema version for nil dirty versions to prevent + // empty schema version for failed down migration on the first migration + // See: https://github.com/golang-migrate/migrate/issues/330 + if version >= 0 || (version == database.NilVersion && dirty) { + query := fmt.Sprintf(`INSERT INTO %s (version, dirty) VALUES (?, ?)`, m.config.MigrationsTable) + if _, err := tx.Exec(query, version, dirty); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } return &database.Error{OrigErr: err, Query: []byte(query)} } } diff --git a/database/sqlite3/sqlite3_test.go b/database/sqlite3/sqlite3_test.go index 7424f4121..b2997566d 100644 --- a/database/sqlite3/sqlite3_test.go +++ b/database/sqlite3/sqlite3_test.go @@ -3,14 +3,17 @@ package sqlite3 import ( "database/sql" "fmt" - "github.com/golang-migrate/migrate/v4" - dt "github.com/golang-migrate/migrate/v4/database/testing" - _ "github.com/golang-migrate/migrate/v4/source/file" - _ "github.com/mattn/go-sqlite3" "io/ioutil" "os" "path/filepath" "testing" + + "github.com/stretchr/testify/assert" + + "github.com/golang-migrate/migrate/v4" + dt "github.com/golang-migrate/migrate/v4/database/testing" + _ "github.com/golang-migrate/migrate/v4/source/file" + _ "github.com/mattn/go-sqlite3" ) func Test(t *testing.T) { @@ -19,15 +22,31 @@ func Test(t *testing.T) { return } defer func() { - os.RemoveAll(dir) + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } }() - fmt.Printf("DB path : %s\n", filepath.Join(dir, "sqlite3.db")) + t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite3.db")) p := &Sqlite{} addr := fmt.Sprintf("sqlite3://%s", filepath.Join(dir, "sqlite3.db")) d, err := p.Open(addr) if err != nil { - t.Fatalf("%v", err) + t.Fatal(err) + } + dt.Test(t, d, []byte("CREATE TABLE t (Qty int, Name string);")) +} + +func TestMigrate(t *testing.T) { + dir, err := ioutil.TempDir("", "sqlite3-driver-test") + if err != nil { + return } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + }() + t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite3.db")) db, err := sql.Open("sqlite3", filepath.Join(dir, "sqlite3.db")) if err != nil { @@ -38,24 +57,127 @@ func Test(t *testing.T) { return } }() - dt.Test(t, d, []byte("CREATE TABLE t (Qty int, Name string);")) driver, err := WithInstance(db, &Config{}) if err != nil { - t.Fatalf("%v", err) + t.Fatal(err) } - if err := d.Drop(); err != nil { + + m, err := migrate.NewWithDatabaseInstance( + "file://./examples/migrations", + "ql", driver) + if err != nil { t.Fatal(err) } + dt.TestMigrate(t, m) +} + +func TestMigrationTable(t *testing.T) { + dir, err := ioutil.TempDir("", "sqlite3-driver-test-migration-table") + if err != nil { + return + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + }() + + t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite3.db")) + + db, err := sql.Open("sqlite3", filepath.Join(dir, "sqlite3.db")) + if err != nil { + return + } + defer func() { + if err := db.Close(); err != nil { + return + } + }() + config := &Config{ + MigrationsTable: "my_migration_table", + } + driver, err := WithInstance(db, config) + if err != nil { + t.Fatal(err) + } m, err := migrate.NewWithDatabaseInstance( - "file://./migration", + "file://./examples/migrations", "ql", driver) if err != nil { - t.Fatalf("%v", err) + t.Fatal(err) } - fmt.Println("UP") + t.Log("UP") err = m.Up() if err != nil { - t.Fatalf("%v", err) + t.Fatal(err) + } + + _, err = db.Query(fmt.Sprintf("SELECT * FROM %s", config.MigrationsTable)) + if err != nil { + t.Fatal(err) + } +} + +func TestNoTxWrap(t *testing.T) { + dir, err := ioutil.TempDir("", "sqlite3-driver-test") + if err != nil { + return + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + }() + t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite3.db")) + p := &Sqlite{} + addr := fmt.Sprintf("sqlite3://%s?x-no-tx-wrap=true", filepath.Join(dir, "sqlite3.db")) + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + // An explicit BEGIN statement would ordinarily fail without x-no-tx-wrap. + // (Transactions in sqlite may not be nested.) + dt.Test(t, d, []byte("BEGIN; CREATE TABLE t (Qty int, Name string); COMMIT;")) +} + +func TestNoTxWrapInvalidValue(t *testing.T) { + dir, err := ioutil.TempDir("", "sqlite3-driver-test") + if err != nil { + return + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + }() + t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite3.db")) + p := &Sqlite{} + addr := fmt.Sprintf("sqlite3://%s?x-no-tx-wrap=yeppers", filepath.Join(dir, "sqlite3.db")) + _, err = p.Open(addr) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "x-no-tx-wrap") + assert.Contains(t, err.Error(), "invalid syntax") + } +} + +func TestMigrateWithDirectoryNameContainsWhitespaces(t *testing.T) { + dir, err := ioutil.TempDir("", "directory name contains whitespaces") + if err != nil { + return + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + }() + dbPath := filepath.Join(dir, "sqlite3.db") + t.Logf("DB path : %s\n", dbPath) + p := &Sqlite{} + addr := fmt.Sprintf("sqlite3://file:%s", dbPath) + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) } + dt.Test(t, d, []byte("CREATE TABLE t (Qty int, Name string);")) } diff --git a/database/sqlserver/README.md b/database/sqlserver/README.md new file mode 100644 index 000000000..c4ef5a3a3 --- /dev/null +++ b/database/sqlserver/README.md @@ -0,0 +1,33 @@ +# Microsoft SQL Server + +`sqlserver://username:password@host/instance?param1=value¶m2=value` +`sqlserver://username:password@host:port?param1=value¶m2=value` + +| URL Query | WithInstance Config | Description | +|------------|---------------------|-------------| +| `x-migrations-table` | `MigrationsTable` | Name of the migrations table | +| `username` | | enter the SQL Server Authentication user id or the Windows Authentication user id in the DOMAIN\User format. On Windows, if user id is empty or missing Single-Sign-On is used. | +| `password` | | The user's password. | +| `host` | | The host to connect to. | +| `port` | | The port to connect to. | +| `instance` | | SQL Server instance name. | +| `database` | `DatabaseName` | The name of the database to connect to | +| `connection+timeout` | | in seconds (default is 0 for no timeout), set to 0 for no timeout. | +| `dial+timeout` | | in seconds (default is 15), set to 0 for no timeout. | +| `encrypt` | | `disable` - Data send between client and server is not encrypted. `false` - Data sent between client and server is not encrypted beyond the login packet (Default). `true` - Data sent between client and server is encrypted. | +| `app+name` || The application name (default is go-mssqldb). | +| `useMsi` | | `true` - Use Azure MSI Authentication for connecting to Sql Server. Must be running from an Azure VM/an instance with MSI enabled. `false` - Use password authentication (Default). See [here for Azure MSI Auth details](https://docs.microsoft.com/en-us/azure/app-service/app-service-web-tutorial-connect-msi). NOTE: Since this cannot be tested locally, this is not officially supported. + +See https://github.com/denisenkom/go-mssqldb for full parameter list. + +## Driver Support + +### Which go-mssqldb driver to us? + +Please note that the deprecated `mssql` driver is not supported. Please use the newer `sqlserver` driver. +See https://github.com/denisenkom/go-mssqldb#deprecated for more information. + +### Official Support by migrate + +Versions of MS SQL Server 2019 newer than CTP3.1 are not officially supported since there are issues testing against the Docker image. +For more info, see: https://github.com/golang-migrate/migrate/issues/160#issuecomment-522433269 diff --git a/database/sqlserver/examples/migrations/1085649617_create_users_table.down.sql b/database/sqlserver/examples/migrations/1085649617_create_users_table.down.sql new file mode 100644 index 000000000..c99ddcdc8 --- /dev/null +++ b/database/sqlserver/examples/migrations/1085649617_create_users_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS users; diff --git a/database/sqlserver/examples/migrations/1085649617_create_users_table.up.sql b/database/sqlserver/examples/migrations/1085649617_create_users_table.up.sql new file mode 100644 index 000000000..92897dcab --- /dev/null +++ b/database/sqlserver/examples/migrations/1085649617_create_users_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE users ( + user_id integer unique, + name varchar(40), + email varchar(40) +); diff --git a/database/sqlserver/examples/migrations/1185749658_add_city_to_users.down.sql b/database/sqlserver/examples/migrations/1185749658_add_city_to_users.down.sql new file mode 100644 index 000000000..940c60712 --- /dev/null +++ b/database/sqlserver/examples/migrations/1185749658_add_city_to_users.down.sql @@ -0,0 +1 @@ +ALTER TABLE users DROP COLUMN IF EXISTS city; diff --git a/database/sqlserver/examples/migrations/1185749658_add_city_to_users.up.sql b/database/sqlserver/examples/migrations/1185749658_add_city_to_users.up.sql new file mode 100644 index 000000000..2add820be --- /dev/null +++ b/database/sqlserver/examples/migrations/1185749658_add_city_to_users.up.sql @@ -0,0 +1,3 @@ +ALTER TABLE users ADD city varchar(100); + + diff --git a/database/sqlserver/examples/migrations/1285849751_add_index_on_user_emails.down.sql b/database/sqlserver/examples/migrations/1285849751_add_index_on_user_emails.down.sql new file mode 100644 index 000000000..3e87dd229 --- /dev/null +++ b/database/sqlserver/examples/migrations/1285849751_add_index_on_user_emails.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS users_email_index; diff --git a/database/sqlserver/examples/migrations/1285849751_add_index_on_user_emails.up.sql b/database/sqlserver/examples/migrations/1285849751_add_index_on_user_emails.up.sql new file mode 100644 index 000000000..03a04639c --- /dev/null +++ b/database/sqlserver/examples/migrations/1285849751_add_index_on_user_emails.up.sql @@ -0,0 +1,3 @@ +CREATE UNIQUE INDEX users_email_index ON users (email); + +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/sqlserver/examples/migrations/1385949617_create_books_table.down.sql b/database/sqlserver/examples/migrations/1385949617_create_books_table.down.sql new file mode 100644 index 000000000..1a0b1a214 --- /dev/null +++ b/database/sqlserver/examples/migrations/1385949617_create_books_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS books; diff --git a/database/sqlserver/examples/migrations/1385949617_create_books_table.up.sql b/database/sqlserver/examples/migrations/1385949617_create_books_table.up.sql new file mode 100644 index 000000000..f1503b518 --- /dev/null +++ b/database/sqlserver/examples/migrations/1385949617_create_books_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE books ( + user_id integer, + name varchar(40), + author varchar(40) +); diff --git a/database/sqlserver/examples/migrations/1485949617_create_movies_table.down.sql b/database/sqlserver/examples/migrations/1485949617_create_movies_table.down.sql new file mode 100644 index 000000000..3a5187689 --- /dev/null +++ b/database/sqlserver/examples/migrations/1485949617_create_movies_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS movies; diff --git a/database/sqlserver/examples/migrations/1485949617_create_movies_table.up.sql b/database/sqlserver/examples/migrations/1485949617_create_movies_table.up.sql new file mode 100644 index 000000000..f0ef5943b --- /dev/null +++ b/database/sqlserver/examples/migrations/1485949617_create_movies_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE movies ( + user_id integer, + name varchar(40), + director varchar(40) +); diff --git a/database/sqlserver/examples/migrations/1585849751_just_a_comment.up.sql b/database/sqlserver/examples/migrations/1585849751_just_a_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/sqlserver/examples/migrations/1585849751_just_a_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/sqlserver/examples/migrations/1685849751_another_comment.up.sql b/database/sqlserver/examples/migrations/1685849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/sqlserver/examples/migrations/1685849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/sqlserver/examples/migrations/1785849751_another_comment.up.sql b/database/sqlserver/examples/migrations/1785849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/sqlserver/examples/migrations/1785849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/sqlserver/examples/migrations/1885849751_another_comment.up.sql b/database/sqlserver/examples/migrations/1885849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/sqlserver/examples/migrations/1885849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/sqlserver/sqlserver.go b/database/sqlserver/sqlserver.go new file mode 100644 index 000000000..024001871 --- /dev/null +++ b/database/sqlserver/sqlserver.go @@ -0,0 +1,402 @@ +package sqlserver + +import ( + "context" + "database/sql" + "fmt" + "io" + "io/ioutil" + nurl "net/url" + "strconv" + "strings" + + "go.uber.org/atomic" + + "github.com/Azure/go-autorest/autorest/adal" + mssql "github.com/denisenkom/go-mssqldb" // mssql support + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database" + "github.com/hashicorp/go-multierror" +) + +func init() { + database.Register("sqlserver", &SQLServer{}) +} + +// DefaultMigrationsTable is the name of the migrations table in the database +var DefaultMigrationsTable = "schema_migrations" + +var ( + ErrNilConfig = fmt.Errorf("no config") + ErrNoDatabaseName = fmt.Errorf("no database name") + ErrNoSchema = fmt.Errorf("no schema") + ErrDatabaseDirty = fmt.Errorf("database is dirty") + ErrMultipleAuthOptionsPassed = fmt.Errorf("both password and useMsi=true were passed.") +) + +var lockErrorMap = map[mssql.ReturnStatus]string{ + -1: "The lock request timed out.", + -2: "The lock request was canceled.", + -3: "The lock request was chosen as a deadlock victim.", + -999: "Parameter validation or other call error.", +} + +// Config for database +type Config struct { + MigrationsTable string + DatabaseName string + SchemaName string +} + +// SQL Server connection +type SQLServer struct { + // Locking and unlocking need to use the same connection + conn *sql.Conn + db *sql.DB + isLocked atomic.Bool + + // Open and WithInstance need to garantuee that config is never nil + config *Config +} + +// WithInstance returns a database instance from an already created database connection. +// +// Note that the deprecated `mssql` driver is not supported. Please use the newer `sqlserver` driver. +func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { + if config == nil { + return nil, ErrNilConfig + } + + if err := instance.Ping(); err != nil { + return nil, err + } + + if config.DatabaseName == "" { + query := `SELECT DB_NAME()` + var databaseName string + if err := instance.QueryRow(query).Scan(&databaseName); err != nil { + return nil, &database.Error{OrigErr: err, Query: []byte(query)} + } + + if len(databaseName) == 0 { + return nil, ErrNoDatabaseName + } + + config.DatabaseName = databaseName + } + + if config.SchemaName == "" { + query := `SELECT SCHEMA_NAME()` + var schemaName string + if err := instance.QueryRow(query).Scan(&schemaName); err != nil { + return nil, &database.Error{OrigErr: err, Query: []byte(query)} + } + + if len(schemaName) == 0 { + return nil, ErrNoSchema + } + + config.SchemaName = schemaName + } + + if len(config.MigrationsTable) == 0 { + config.MigrationsTable = DefaultMigrationsTable + } + + conn, err := instance.Conn(context.Background()) + + if err != nil { + return nil, err + } + + ss := &SQLServer{ + conn: conn, + db: instance, + config: config, + } + + if err := ss.ensureVersionTable(); err != nil { + return nil, err + } + + return ss, nil +} + +// Open a connection to the database. +func (ss *SQLServer) Open(url string) (database.Driver, error) { + purl, err := nurl.Parse(url) + if err != nil { + return nil, err + } + + useMsiParam := purl.Query().Get("useMsi") + useMsi := false + if len(useMsiParam) > 0 { + useMsi, err = strconv.ParseBool(useMsiParam) + if err != nil { + return nil, err + } + } + + if _, isPasswordSet := purl.User.Password(); useMsi && isPasswordSet { + return nil, ErrMultipleAuthOptionsPassed + } + + filteredURL := migrate.FilterCustomQuery(purl).String() + + var db *sql.DB + if useMsi { + resource := getAADResourceFromServerUri(purl) + tokenProvider, err := getMSITokenProvider(resource) + if err != nil { + return nil, err + } + + connector, err := mssql.NewAccessTokenConnector( + filteredURL, tokenProvider) + if err != nil { + return nil, err + } + + db = sql.OpenDB(connector) + + } else { + db, err = sql.Open("sqlserver", filteredURL) + if err != nil { + return nil, err + } + } + + migrationsTable := purl.Query().Get("x-migrations-table") + + px, err := WithInstance(db, &Config{ + DatabaseName: purl.Path, + MigrationsTable: migrationsTable, + }) + + if err != nil { + return nil, err + } + + return px, nil +} + +// Close the database connection +func (ss *SQLServer) Close() error { + connErr := ss.conn.Close() + dbErr := ss.db.Close() + if connErr != nil || dbErr != nil { + return fmt.Errorf("conn: %v, db: %v", connErr, dbErr) + } + return nil +} + +// Lock creates an advisory local on the database to prevent multiple migrations from running at the same time. +func (ss *SQLServer) Lock() error { + return database.CasRestoreOnErr(&ss.isLocked, false, true, database.ErrLocked, func() error { + aid, err := database.GenerateAdvisoryLockId(ss.config.DatabaseName, ss.config.SchemaName) + if err != nil { + return err + } + + // This will either obtain the lock immediately and return true, + // or return false if the lock cannot be acquired immediately. + // MS Docs: sp_getapplock: https://docs.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-getapplock-transact-sql?view=sql-server-2017 + query := `EXEC sp_getapplock @Resource = @p1, @LockMode = 'Update', @LockOwner = 'Session', @LockTimeout = 0` + + var status mssql.ReturnStatus + if _, err = ss.conn.ExecContext(context.Background(), query, aid, &status); err == nil && status > -1 { + return nil + } else if err != nil { + return &database.Error{OrigErr: err, Err: "try lock failed", Query: []byte(query)} + } else { + return &database.Error{Err: fmt.Sprintf("try lock failed with error %v: %v", status, lockErrorMap[status]), Query: []byte(query)} + } + }) +} + +// Unlock froms the migration lock from the database +func (ss *SQLServer) Unlock() error { + return database.CasRestoreOnErr(&ss.isLocked, true, false, database.ErrNotLocked, func() error { + aid, err := database.GenerateAdvisoryLockId(ss.config.DatabaseName, ss.config.SchemaName) + if err != nil { + return err + } + + // MS Docs: sp_releaseapplock: https://docs.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-releaseapplock-transact-sql?view=sql-server-2017 + query := `EXEC sp_releaseapplock @Resource = @p1, @LockOwner = 'Session'` + if _, err := ss.conn.ExecContext(context.Background(), query, aid); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + return nil + }) +} + +// Run the migrations for the database +func (ss *SQLServer) Run(migration io.Reader) error { + migr, err := ioutil.ReadAll(migration) + if err != nil { + return err + } + + // run migration + query := string(migr[:]) + if _, err := ss.conn.ExecContext(context.Background(), query); err != nil { + if msErr, ok := err.(mssql.Error); ok { + message := fmt.Sprintf("migration failed: %s", msErr.Message) + if msErr.ProcName != "" { + message = fmt.Sprintf("%s (proc name %s)", msErr.Message, msErr.ProcName) + } + return database.Error{OrigErr: err, Err: message, Query: migr, Line: uint(msErr.LineNo)} + } + return database.Error{OrigErr: err, Err: "migration failed", Query: migr} + } + + return nil +} + +// SetVersion for the current database +func (ss *SQLServer) SetVersion(version int, dirty bool) error { + + tx, err := ss.conn.BeginTx(context.Background(), &sql.TxOptions{}) + if err != nil { + return &database.Error{OrigErr: err, Err: "transaction start failed"} + } + + query := `TRUNCATE TABLE "` + ss.config.MigrationsTable + `"` + if _, err := tx.Exec(query); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + // Also re-write the schema version for nil dirty versions to prevent + // empty schema version for failed down migration on the first migration + // See: https://github.com/golang-migrate/migrate/issues/330 + if version >= 0 || (version == database.NilVersion && dirty) { + var dirtyBit int + if dirty { + dirtyBit = 1 + } + query = `INSERT INTO "` + ss.config.MigrationsTable + `" (version, dirty) VALUES (@p1, @p2)` + if _, err := tx.Exec(query, version, dirtyBit); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + + if err := tx.Commit(); err != nil { + return &database.Error{OrigErr: err, Err: "transaction commit failed"} + } + + return nil +} + +// Version of the current database state +func (ss *SQLServer) Version() (version int, dirty bool, err error) { + query := `SELECT TOP 1 version, dirty FROM "` + ss.config.MigrationsTable + `"` + err = ss.conn.QueryRowContext(context.Background(), query).Scan(&version, &dirty) + switch { + case err == sql.ErrNoRows: + return database.NilVersion, false, nil + + case err != nil: + // FIXME: convert to MSSQL error + return 0, false, &database.Error{OrigErr: err, Query: []byte(query)} + + default: + return version, dirty, nil + } +} + +// Drop all tables from the database. +func (ss *SQLServer) Drop() error { + + // drop all referential integrity constraints + query := ` + DECLARE @Sql NVARCHAR(500) DECLARE @Cursor CURSOR + + SET @Cursor = CURSOR FAST_FORWARD FOR + SELECT DISTINCT sql = 'ALTER TABLE [' + tc2.TABLE_NAME + '] DROP [' + rc1.CONSTRAINT_NAME + ']' + FROM INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS rc1 + LEFT JOIN INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc2 ON tc2.CONSTRAINT_NAME =rc1.CONSTRAINT_NAME + + OPEN @Cursor FETCH NEXT FROM @Cursor INTO @Sql + + WHILE (@@FETCH_STATUS = 0) + BEGIN + Exec sp_executesql @Sql + FETCH NEXT FROM @Cursor INTO @Sql + END + + CLOSE @Cursor DEALLOCATE @Cursor` + + if _, err := ss.conn.ExecContext(context.Background(), query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + // drop the tables + query = `EXEC sp_MSforeachtable 'DROP TABLE ?'` + if _, err := ss.conn.ExecContext(context.Background(), query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + return nil +} + +func (ss *SQLServer) ensureVersionTable() (err error) { + if err = ss.Lock(); err != nil { + return err + } + + defer func() { + if e := ss.Unlock(); e != nil { + if err == nil { + err = e + } else { + err = multierror.Append(err, e) + } + } + }() + + query := `IF NOT EXISTS + (SELECT * + FROM sysobjects + WHERE id = object_id(N'[dbo].[` + ss.config.MigrationsTable + `]') + AND OBJECTPROPERTY(id, N'IsUserTable') = 1 + ) + CREATE TABLE ` + ss.config.MigrationsTable + ` ( version BIGINT PRIMARY KEY NOT NULL, dirty BIT NOT NULL );` + + if _, err = ss.conn.ExecContext(context.Background(), query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + return nil +} + +func getMSITokenProvider(resource string) (func() (string, error), error) { + msi, err := adal.NewServicePrincipalTokenFromManagedIdentity(resource, nil) + if err != nil { + return nil, err + } + + return func() (string, error) { + err := msi.EnsureFresh() + if err != nil { + return "", err + } + token := msi.OAuthToken() + return token, nil + }, nil +} + +// The sql server resource can change across clouds so get it +// dynamically based on the server uri. +// ex. .database.windows.net -> https://database.windows.net +func getAADResourceFromServerUri(purl *nurl.URL) string { + return fmt.Sprintf("%s%s", "https://", strings.Join(strings.Split(purl.Hostname(), ".")[1:], ".")) +} diff --git a/database/sqlserver/sqlserver_test.go b/database/sqlserver/sqlserver_test.go new file mode 100644 index 000000000..ad0dc79ed --- /dev/null +++ b/database/sqlserver/sqlserver_test.go @@ -0,0 +1,291 @@ +package sqlserver + +import ( + "context" + "database/sql" + sqldriver "database/sql/driver" + "fmt" + "log" + "strings" + "testing" + "time" + + "github.com/dhui/dktest" + "github.com/golang-migrate/migrate/v4" + + dt "github.com/golang-migrate/migrate/v4/database/testing" + "github.com/golang-migrate/migrate/v4/dktesting" + + _ "github.com/golang-migrate/migrate/v4/source/file" +) + +const defaultPort = 1433 +const saPassword = "Root1234" + +var ( + opts = dktest.Options{ + Env: map[string]string{"ACCEPT_EULA": "Y", "SA_PASSWORD": saPassword, "MSSQL_PID": "Express"}, + PortRequired: true, ReadyFunc: isReady, PullTimeout: 2 * time.Minute, + } + // Container versions: https://mcr.microsoft.com/v2/mssql/server/tags/list + specs = []dktesting.ContainerSpec{ + {ImageName: "mcr.microsoft.com/mssql/server:2017-latest", Options: opts}, + {ImageName: "mcr.microsoft.com/mssql/server:2019-latest", Options: opts}, + } +) + +func msConnectionString(host, port string) string { + return fmt.Sprintf("sqlserver://sa:%v@%v:%v?database=master", saPassword, host, port) +} + +func msConnectionStringMsiWithPassword(host, port string, useMsi bool) string { + return fmt.Sprintf("sqlserver://sa:%v@%v:%v?database=master&useMsi=%t", saPassword, host, port, useMsi) +} + +func msConnectionStringMsi(host, port string, useMsi bool) string { + return fmt.Sprintf("sqlserver://sa@%v:%v?database=master&useMsi=%t", host, port, useMsi) +} + +func isReady(ctx context.Context, c dktest.ContainerInfo) bool { + ip, port, err := c.Port(defaultPort) + if err != nil { + return false + } + uri := msConnectionString(ip, port) + db, err := sql.Open("sqlserver", uri) + if err != nil { + return false + } + defer func() { + if err := db.Close(); err != nil { + log.Println("close error:", err) + } + }() + if err = db.PingContext(ctx); err != nil { + switch err { + case sqldriver.ErrBadConn: + return false + default: + fmt.Println(err) + } + return false + } + + return true +} + +func Test(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } + + addr := msConnectionString(ip, port) + p := &SQLServer{} + d, err := p.Open(addr) + if err != nil { + t.Fatalf("%v", err) + } + + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + + dt.Test(t, d, []byte("SELECT 1")) + }) +} + +func TestMigrate(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } + + addr := msConnectionString(ip, port) + p := &SQLServer{} + d, err := p.Open(addr) + if err != nil { + t.Fatalf("%v", err) + } + + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + + m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "master", d) + if err != nil { + t.Fatal(err) + } + dt.TestMigrate(t, m) + }) +} + +func TestMultiStatement(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := msConnectionString(ip, port) + ms := &SQLServer{} + d, err := ms.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + if err := d.Run(strings.NewReader("CREATE TABLE foo (foo text); CREATE TABLE bar (bar text);")); err != nil { + t.Fatalf("expected err to be nil, got %v", err) + } + + // make sure second table exists + var exists int + if err := d.(*SQLServer).conn.QueryRowContext(context.Background(), "SELECT COUNT(1) FROM information_schema.tables WHERE table_name = 'bar' AND table_schema = (SELECT schema_name()) AND table_catalog = (SELECT db_name())").Scan(&exists); err != nil { + t.Fatal(err) + } + if exists != 1 { + t.Fatalf("expected table bar to exist") + } + }) +} + +func TestErrorParsing(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.FirstPort() + if err != nil { + t.Fatal(err) + } + + addr := msConnectionString(ip, port) + p := &SQLServer{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + + wantErr := `migration failed: Unknown object type 'TABLEE' used in a CREATE, DROP, or ALTER statement. in line 1:` + + ` CREATE TABLE foo (foo text); CREATE TABLEE bar (bar text); (details: mssql: Unknown object type ` + + `'TABLEE' used in a CREATE, DROP, or ALTER statement.)` + if err := d.Run(strings.NewReader("CREATE TABLE foo (foo text); CREATE TABLEE bar (bar text);")); err == nil { + t.Fatal("expected err but got nil") + } else if err.Error() != wantErr { + t.Fatalf("expected '%s' but got '%s'", wantErr, err.Error()) + } + }) +} + +func TestLockWorks(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } + + addr := fmt.Sprintf("sqlserver://sa:%v@%v:%v?master", saPassword, ip, port) + p := &SQLServer{} + d, err := p.Open(addr) + if err != nil { + t.Fatalf("%v", err) + } + dt.Test(t, d, []byte("SELECT 1")) + + ms := d.(*SQLServer) + + err = ms.Lock() + if err != nil { + t.Fatal(err) + } + err = ms.Unlock() + if err != nil { + t.Fatal(err) + } + + // make sure the 2nd lock works (RELEASE_LOCK is very finicky) + err = ms.Lock() + if err != nil { + t.Fatal(err) + } + err = ms.Unlock() + if err != nil { + t.Fatal(err) + } + }) +} + +func TestMsiTrue(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } + + addr := msConnectionStringMsi(ip, port, true) + p := &SQLServer{} + _, err = p.Open(addr) + if err == nil { + t.Fatal("MSI should fail when not running in an Azure context.") + } + }) +} + +func TestOpenWithPasswordAndMSI(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } + + addr := msConnectionStringMsiWithPassword(ip, port, true) + p := &SQLServer{} + _, err = p.Open(addr) + if err == nil { + t.Fatal("Open should fail when both password and useMsi=true are passed.") + } + + addr = msConnectionStringMsiWithPassword(ip, port, false) + p = &SQLServer{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + + dt.Test(t, d, []byte("SELECT 1")) + }) +} + +func TestMsiFalse(t *testing.T) { + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } + + addr := msConnectionStringMsi(ip, port, false) + p := &SQLServer{} + _, err = p.Open(addr) + if err == nil { + t.Fatal("Open should fail since no password was passed and useMsi is false.") + } + }) +} diff --git a/database/stub/stub.go b/database/stub/stub.go index 4ad0193bd..238ce8ba6 100644 --- a/database/stub/stub.go +++ b/database/stub/stub.go @@ -1,6 +1,7 @@ package stub import ( + "go.uber.org/atomic" "io" "io/ioutil" "reflect" @@ -19,7 +20,7 @@ type Stub struct { MigrationSequence []string LastRunMigration []byte // todo: make []string IsDirty bool - IsLocked bool + isLocked atomic.Bool Config *Config } @@ -27,7 +28,7 @@ type Stub struct { func (s *Stub) Open(url string) (database.Driver, error) { return &Stub{ Url: url, - CurrentVersion: -1, + CurrentVersion: database.NilVersion, MigrationSequence: make([]string, 0), Config: &Config{}, }, nil @@ -38,7 +39,7 @@ type Config struct{} func WithInstance(instance interface{}, config *Config) (database.Driver, error) { return &Stub{ Instance: instance, - CurrentVersion: -1, + CurrentVersion: database.NilVersion, MigrationSequence: make([]string, 0), Config: config, }, nil @@ -49,15 +50,16 @@ func (s *Stub) Close() error { } func (s *Stub) Lock() error { - if s.IsLocked { + if !s.isLocked.CAS(false, true) { return database.ErrLocked } - s.IsLocked = true return nil } func (s *Stub) Unlock() error { - s.IsLocked = false + if !s.isLocked.CAS(true, false) { + return database.ErrNotLocked + } return nil } @@ -84,7 +86,7 @@ func (s *Stub) Version() (version int, dirty bool, err error) { const DROP = "DROP" func (s *Stub) Drop() error { - s.CurrentVersion = -1 + s.CurrentVersion = database.NilVersion s.LastRunMigration = nil s.MigrationSequence = append(s.MigrationSequence, DROP) return nil diff --git a/database/stub/stub_test.go b/database/stub/stub_test.go index 2b966daa1..131c935a8 100644 --- a/database/stub/stub_test.go +++ b/database/stub/stub_test.go @@ -1,6 +1,9 @@ package stub import ( + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/source" + "github.com/golang-migrate/migrate/v4/source/stub" "testing" dt "github.com/golang-migrate/migrate/v4/database/testing" @@ -14,3 +17,27 @@ func Test(t *testing.T) { } dt.Test(t, d, []byte("/* foobar migration */")) } + +func TestMigrate(t *testing.T) { + s := &Stub{} + d, err := s.Open("") + if err != nil { + t.Fatal(err) + } + + stubMigrations := source.NewMigrations() + stubMigrations.Append(&source.Migration{Version: 1, Direction: source.Up, Identifier: "CREATE 1"}) + stubMigrations.Append(&source.Migration{Version: 1, Direction: source.Down, Identifier: "DROP 1"}) + src := &stub.Stub{} + srcDrv, err := src.Open("") + if err != nil { + t.Fatal(err) + } + srcDrv.(*stub.Stub).Migrations = stubMigrations + m, err := migrate.NewWithInstance("stub", srcDrv, "", d) + if err != nil { + t.Fatal(err) + } + + dt.TestMigrate(t, m) +} diff --git a/database/testing/migrate_testing.go b/database/testing/migrate_testing.go new file mode 100644 index 000000000..be8ed195f --- /dev/null +++ b/database/testing/migrate_testing.go @@ -0,0 +1,34 @@ +// Package testing has the database tests. +// All database drivers must pass the Test function. +// This lives in it's own package so it stays a test dependency. +package testing + +import ( + "testing" +) + +import ( + "github.com/golang-migrate/migrate/v4" +) + +// TestMigrate runs integration-tests between the Migrate layer and database implementations. +func TestMigrate(t *testing.T, m *migrate.Migrate) { + TestMigrateUp(t, m) + TestMigrateDrop(t, m) +} + +// Regression test for preventing a regression for #164 https://github.com/golang-migrate/migrate/pull/173 +// Similar to TestDrop(), but tests the dropping mechanism through the Migrate logic instead, to check for +// double-locking during the Drop logic. +func TestMigrateDrop(t *testing.T, m *migrate.Migrate) { + if err := m.Drop(); err != nil { + t.Fatal(err) + } +} + +func TestMigrateUp(t *testing.T, m *migrate.Migrate) { + t.Log("UP") + if err := m.Up(); err != nil { + t.Fatal(err) + } +} diff --git a/database/testing/testing.go b/database/testing/testing.go index 01474836c..bd3294b1e 100644 --- a/database/testing/testing.go +++ b/database/testing/testing.go @@ -5,6 +5,7 @@ package testing import ( "bytes" + "errors" "fmt" "io" "testing" @@ -16,14 +17,15 @@ import ( // Test runs tests against database implementations. func Test(t *testing.T, d database.Driver, migration []byte) { if migration == nil { - panic("test must provide migration reader") + t.Fatal("test must provide migration reader") } TestNilVersion(t, d) // test first TestLockAndUnlock(t, d) TestRun(t, d, bytes.NewReader(migration)) - TestDrop(t, d) TestSetVersion(t, d) // also tests Version() + // Drop breaks the driver, so test it last. + TestDrop(t, d) } func TestNilVersion(t *testing.T, d database.Driver) { @@ -38,7 +40,9 @@ func TestNilVersion(t *testing.T, d database.Driver) { func TestLockAndUnlock(t *testing.T, d database.Driver) { // add a timeout, in case there is a deadlock - done := make(chan bool, 1) + done := make(chan struct{}) + errs := make(chan error) + go func() { timeout := time.After(15 * time.Second) for { @@ -46,42 +50,58 @@ func TestLockAndUnlock(t *testing.T, d database.Driver) { case <-done: return case <-timeout: - panic(fmt.Sprintf("Timeout after 15 seconds. Looks like a deadlock in Lock/UnLock.\n%#v", d)) + errs <- fmt.Errorf("Timeout after 15 seconds. Looks like a deadlock in Lock/UnLock.\n%#v", d) + return } } }() - defer func() { - done <- true - }() // run the locking test ... + go func() { + if err := d.Lock(); err != nil { + errs <- err + return + } - if err := d.Lock(); err != nil { - t.Fatal(err) - } + // try to acquire lock again + if err := d.Lock(); err == nil { + errs <- errors.New("lock: expected err not to be nil") + return + } - // try to acquire lock again - if err := d.Lock(); err == nil { - t.Fatal("Lock: expected err not to be nil") - } + // unlock + if err := d.Unlock(); err != nil { + errs <- err + return + } - // unlock - if err := d.Unlock(); err != nil { - t.Fatal(err) - } + // try to lock + if err := d.Lock(); err != nil { + errs <- err + return + } + if err := d.Unlock(); err != nil { + errs <- err + return + } + // notify everyone + close(done) + }() - // try to lock - if err := d.Lock(); err != nil { - t.Fatal(err) - } - if err := d.Unlock(); err != nil { - t.Fatal(err) + // wait for done or any error + for { + select { + case <-done: + return + case err := <-errs: + t.Fatal(err) + } } } func TestRun(t *testing.T, d database.Driver, migration io.Reader) { if migration == nil { - panic("migration can't be nil") + t.Fatal("migration can't be nil") } if err := d.Run(migration); err != nil { @@ -96,43 +116,40 @@ func TestDrop(t *testing.T, d database.Driver) { } func TestSetVersion(t *testing.T, d database.Driver) { - if err := d.SetVersion(1, true); err != nil { - t.Fatal(err) - } - - // call again - if err := d.SetVersion(1, true); err != nil { - t.Fatal(err) - } - - v, dirty, err := d.Version() - if err != nil { - t.Fatal(err) - } - if !dirty { - t.Fatal("expected dirty") - } - if v != 1 { - t.Fatal("expected version to be 1") - } - - if err := d.SetVersion(2, false); err != nil { - t.Fatal(err) - } - - // call again - if err := d.SetVersion(2, false); err != nil { - t.Fatal(err) - } - - v, dirty, err = d.Version() - if err != nil { - t.Fatal(err) - } - if dirty { - t.Fatal("expected not dirty") - } - if v != 2 { - t.Fatal("expected version to be 2") + // nolint:maligned + testCases := []struct { + name string + version int + dirty bool + expectedErr error + expectedReadErr error + expectedVersion int + expectedDirty bool + }{ + {name: "set 1 dirty", version: 1, dirty: true, expectedErr: nil, expectedReadErr: nil, expectedVersion: 1, expectedDirty: true}, + {name: "re-set 1 dirty", version: 1, dirty: true, expectedErr: nil, expectedReadErr: nil, expectedVersion: 1, expectedDirty: true}, + {name: "set 2 clean", version: 2, dirty: false, expectedErr: nil, expectedReadErr: nil, expectedVersion: 2, expectedDirty: false}, + {name: "re-set 2 clean", version: 2, dirty: false, expectedErr: nil, expectedReadErr: nil, expectedVersion: 2, expectedDirty: false}, + {name: "last migration dirty", version: database.NilVersion, dirty: true, expectedErr: nil, expectedReadErr: nil, expectedVersion: database.NilVersion, expectedDirty: true}, + {name: "last migration clean", version: database.NilVersion, dirty: false, expectedErr: nil, expectedReadErr: nil, expectedVersion: database.NilVersion, expectedDirty: false}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := d.SetVersion(tc.version, tc.dirty) + if err != tc.expectedErr { + t.Fatal("Got unexpected error:", err, "!=", tc.expectedErr) + } + v, dirty, readErr := d.Version() + if readErr != tc.expectedReadErr { + t.Fatal("Got unexpected error:", readErr, "!=", tc.expectedReadErr) + } + if v != tc.expectedVersion { + t.Error("Got unexpected version:", v, "!=", tc.expectedVersion) + } + if dirty != tc.expectedDirty { + t.Error("Got unexpected dirty value:", dirty, "!=", tc.dirty) + } + }) } } diff --git a/database/util.go b/database/util.go index 7de1d1b6a..de66d5b80 100644 --- a/database/util.go +++ b/database/util.go @@ -2,14 +2,32 @@ package database import ( "fmt" + "go.uber.org/atomic" "hash/crc32" + "strings" ) -const advisoryLockIdSalt uint = 1486364155 +const advisoryLockIDSalt uint = 1486364155 // GenerateAdvisoryLockId inspired by rails migrations, see https://goo.gl/8o9bCT -func GenerateAdvisoryLockId(databaseName string) (string, error) { +func GenerateAdvisoryLockId(databaseName string, additionalNames ...string) (string, error) { // nolint: golint + if len(additionalNames) > 0 { + databaseName = strings.Join(append(additionalNames, databaseName), "\x00") + } sum := crc32.ChecksumIEEE([]byte(databaseName)) - sum = sum * uint32(advisoryLockIdSalt) - return fmt.Sprintf("%v", sum), nil + sum = sum * uint32(advisoryLockIDSalt) + return fmt.Sprint(sum), nil +} + +// CasRestoreOnErr CAS wrapper to automatically restore the lock state on error +func CasRestoreOnErr(lock *atomic.Bool, o, n bool, casErr error, f func() error) error { + if !lock.CAS(o, n) { + return casErr + } + if err := f(); err != nil { + // Automatically unlock/lock on error + lock.Store(o) + return err + } + return nil } diff --git a/database/util_test.go b/database/util_test.go index 0b66d2d97..3f1dc73ae 100644 --- a/database/util_test.go +++ b/database/util_test.go @@ -1,20 +1,41 @@ package database import ( + "errors" + "go.uber.org/atomic" "testing" ) func TestGenerateAdvisoryLockId(t *testing.T) { testcases := []struct { dbname string + additional []string expectedID string // empty string signifies that an error is expected }{ - {dbname: "database_name", expectedID: "1764327054"}, + { + dbname: "database_name", + expectedID: "1764327054", + }, + { + dbname: "database_name", + additional: []string{"schema_name_1"}, + expectedID: "2453313553", + }, + { + dbname: "database_name", + additional: []string{"schema_name_2"}, + expectedID: "235207038", + }, + { + dbname: "database_name", + additional: []string{"schema_name_1", "schema_name_2"}, + expectedID: "3743845847", + }, } for _, tc := range testcases { t.Run(tc.dbname, func(t *testing.T) { - if id, err := GenerateAdvisoryLockId("database_name"); err == nil { + if id, err := GenerateAdvisoryLockId(tc.dbname, tc.additional...); err == nil { if id != tc.expectedID { t.Error("Generated incorrect ID:", id, "!=", tc.expectedID) } @@ -26,3 +47,60 @@ func TestGenerateAdvisoryLockId(t *testing.T) { }) } } + +func TestCasRestoreOnErr(t *testing.T) { + casErr := errors.New("test lock CAS failure") + fErr := errors.New("test callback error") + + testcases := []struct { + name string + lock *atomic.Bool + from bool + to bool + expectLock bool + fErr error + expectError error + }{ + { + name: "Test positive CAS lock", + lock: atomic.NewBool(false), + from: false, + to: true, + expectLock: true, + fErr: nil, + expectError: nil, + }, + { + name: "Test negative CAS lock", + lock: atomic.NewBool(true), + from: false, + to: true, + expectLock: true, + fErr: nil, + expectError: casErr, + }, + { + name: "Test negative with callback lock", + lock: atomic.NewBool(false), + from: false, + to: true, + expectLock: false, + fErr: fErr, + expectError: fErr, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + if err := CasRestoreOnErr(tc.lock, tc.from, tc.to, casErr, func() error { + return tc.fErr + }); err != tc.expectError { + t.Error("Incorrect error value returned") + } + + if tc.lock.Load() != tc.expectLock { + t.Error("Incorrect state of lock") + } + }) + } +} diff --git a/dktesting/dktesting.go b/dktesting/dktesting.go new file mode 100644 index 000000000..342af2fdc --- /dev/null +++ b/dktesting/dktesting.go @@ -0,0 +1,35 @@ +package dktesting + +import ( + "testing" +) + +import ( + "github.com/dhui/dktest" +) + +// ContainerSpec holds Docker testing setup specifications +type ContainerSpec struct { + ImageName string + Options dktest.Options +} + +// ParallelTest runs Docker tests in parallel +func ParallelTest(t *testing.T, specs []ContainerSpec, + testFunc func(*testing.T, dktest.ContainerInfo)) { + + for i, spec := range specs { + spec := spec // capture range variable, see https://goo.gl/60w3p2 + + // Only test against one version in short mode + // TODO: order is random, maybe always pick first version instead? + if i > 0 && testing.Short() { + t.Logf("Skipping %v in short mode", spec.ImageName) + } else { + t.Run(spec.ImageName, func(t *testing.T) { + t.Parallel() + dktest.Run(t, spec.ImageName, spec.Options, testFunc) + }) + } + } +} diff --git a/dktesting/example_test.go b/dktesting/example_test.go new file mode 100644 index 000000000..5f5ccaac1 --- /dev/null +++ b/dktesting/example_test.go @@ -0,0 +1,30 @@ +package dktesting_test + +import ( + "context" + "testing" +) + +import ( + "github.com/dhui/dktest" +) + +import ( + "github.com/golang-migrate/migrate/v4/dktesting" +) + +func ExampleParallelTest() { + t := &testing.T{} // Should actually be used in a Test + + var isReady = func(ctx context.Context, c dktest.ContainerInfo) bool { + // Return true if the container is ready to run tests. + // Don't block here though. Use the Context to timeout container ready checks. + return true + } + + dktesting.ParallelTest(t, []dktesting.ContainerSpec{{ImageName: "docker_image:9.6", + Options: dktest.Options{ReadyFunc: isReady}}}, func(t *testing.T, c dktest.ContainerInfo) { + // Run your test/s ... + t.Fatal("...") + }) +} diff --git a/go.mod b/go.mod index a49077645..c926ae588 100644 --- a/go.mod +++ b/go.mod @@ -1,43 +1,74 @@ module github.com/golang-migrate/migrate/v4 require ( - cloud.google.com/go v0.27.0 - github.com/Microsoft/go-winio v0.4.11 // indirect - github.com/Sirupsen/logrus v1.0.6 // indirect - github.com/aws/aws-sdk-go v1.15.34 - github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 // indirect - github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect - github.com/cockroachdb/cockroach-go v0.0.0-20180212155653-59c0560478b7 - github.com/cznic/ql v1.2.0 - github.com/docker/distribution v0.0.0-20180720172123-0dae0957e5fe // indirect - github.com/docker/docker v0.0.0-20170502054910-90d35abf7b35 - github.com/docker/go-connections v0.4.0 // indirect - github.com/docker/go-units v0.3.3 // indirect - github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect - github.com/fsouza/fake-gcs-server v1.2.0 - github.com/go-ini/ini v1.38.2 // indirect - github.com/go-sql-driver/mysql v1.4.0 - github.com/gocql/gocql v0.0.0-20180913072538-864d5908455a - github.com/google/go-github v17.0.0+incompatible - github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 // indirect - github.com/google/martian v2.1.0+incompatible // indirect - github.com/gorilla/context v1.1.1 // indirect - github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect - github.com/kshvakov/clickhouse v1.3.4 - github.com/lib/pq v1.0.0 - github.com/mattn/go-sqlite3 v1.9.0 - github.com/onsi/gomega v1.4.2 // indirect - github.com/opencontainers/go-digest v1.0.0-rc1 // indirect - github.com/sirupsen/logrus v1.2.0 + cloud.google.com/go/spanner v1.24.0 + cloud.google.com/go/storage v1.10.0 + github.com/Azure/go-autorest/autorest/adal v0.9.16 + github.com/ClickHouse/clickhouse-go v1.4.3 + github.com/Microsoft/go-winio v0.5.0 // indirect + github.com/apache/arrow/go/arrow v0.0.0-20211013220434-5962184e7a30 // indirect + github.com/aws/aws-sdk-go v1.17.7 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.4 // indirect + github.com/cenkalti/backoff/v4 v4.0.2 + github.com/cockroachdb/cockroach-go/v2 v2.1.1 + github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369 // indirect + github.com/denisenkom/go-mssqldb v0.10.0 + github.com/dhui/dktest v0.3.7 + github.com/docker/docker v20.10.9+incompatible + github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712 // indirect + github.com/fsouza/fake-gcs-server v1.17.0 + github.com/gabriel-vasile/mimetype v1.4.0 // indirect + github.com/go-sql-driver/mysql v1.5.0 + github.com/gobuffalo/here v0.6.0 + github.com/gocql/gocql v0.0.0-20210515062232-b7ef815b4556 + github.com/gofrs/uuid v4.0.0+incompatible // indirect + github.com/golang-jwt/jwt/v4 v4.1.0 // indirect + github.com/google/go-github/v39 v39.2.0 + github.com/gorilla/mux v1.7.4 // indirect + github.com/hashicorp/go-multierror v1.1.0 + github.com/jackc/pgconn v1.8.0 + github.com/jackc/pgerrcode v0.0.0-20201024163028-a0d42d470451 + github.com/jackc/pgproto3/v2 v2.0.7 // indirect + github.com/jackc/pgx/v4 v4.10.1 + github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect + github.com/klauspost/compress v1.13.6 // indirect + github.com/ktrysmt/go-bitbucket v0.6.4 + github.com/lib/pq v1.10.0 + github.com/markbates/pkger v0.15.1 + github.com/mattn/go-sqlite3 v1.14.6 + github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect + github.com/mutecomm/go-sqlcipher/v4 v4.4.0 + github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8 + github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba + github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect + github.com/sirupsen/logrus v1.8.1 + github.com/snowflakedb/gosnowflake v1.6.3 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.6.2 - go.opencensus.io v0.16.0 // indirect - golang.org/x/net v0.0.0-20190522155817-f3200d17e092 - golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 - google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf - google.golang.org/genproto v0.0.0-20180912233945-5a2fd4cab2d6 - gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect - gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect + github.com/spf13/viper v1.4.0 + github.com/stretchr/testify v1.7.0 + github.com/xanzy/go-gitlab v0.15.0 + gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b // indirect + go.mongodb.org/mongo-driver v1.7.0 + go.uber.org/atomic v1.6.0 + golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect + golang.org/x/net v0.0.0-20211013171255-e13a2654a71e // indirect + golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914 + golang.org/x/sys v0.0.0-20211013075003-97ac67df715c // indirect + golang.org/x/tools v0.1.5 + google.golang.org/api v0.51.0 + google.golang.org/genproto v0.0.0-20211013025323-ce878158c4d4 + google.golang.org/grpc v1.41.0 // indirect + modernc.org/b v1.0.0 // indirect + modernc.org/db v1.0.0 // indirect + modernc.org/file v1.0.0 // indirect + modernc.org/fileutil v1.0.0 // indirect + modernc.org/golex v1.0.0 // indirect + modernc.org/internal v1.0.0 // indirect + modernc.org/lldb v1.0.0 // indirect + modernc.org/ql v1.0.0 + modernc.org/sortutil v1.1.0 // indirect + modernc.org/sqlite v1.10.6 + modernc.org/zappy v1.0.0 // indirect ) -go 1.13 +go 1.16 diff --git a/go.sum b/go.sum index 878a7a395..be52c0f75 100644 --- a/go.sum +++ b/go.sum @@ -1,284 +1,1928 @@ +bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898 h1:SC+c6A1qTFstO9qmB86mPV2IpYme/2ZoEQ0hrP+wo+Q= +bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.27.0 h1:Xa8ZWro6QYKOwDKtxfKsiE0ea2jD39nx32RxtF5RjYE= -cloud.google.com/go v0.27.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -contrib.go.opencensus.io/exporter/stackdriver v0.6.0 h1:U0FQWsZU3aO8W+BrZc88T8fdd24qe3Phawa9V9oaVUE= -contrib.go.opencensus.io/exporter/stackdriver v0.6.0/go.mod h1:QeFzMJDAw8TXt5+aRaSuE8l5BwaMIOIlaVkBOPRuMuw= -git.apache.org/thrift.git v0.0.0-20180807212849-6e67faa92827/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.88.0 h1:MZ2cf9Elnv1wqccq8ooKO2MqHQLc+ChCp/+QWObCpxg= +cloud.google.com/go v0.88.0/go.mod h1:dnKwfYbP9hQhefiUvpbcAyoGSHUrOxR20JVElLiUvEY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0 h1:PQcPefKFdaIzjQFbiyOgAqyx8q5djaE7x9Sqe712DPA= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1 h1:ukjixP1wl0LpnZ6LWtZJ0mX5tBmjp1f8Sqer8Z2OMUU= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/spanner v1.24.0 h1:LDLFxHGdlBK8m5i8fOEqEPRTNJunKfxhCvUHAlS3PpM= +cloud.google.com/go/spanner v1.24.0/go.mod h1:EZI0yH1D/PrXK0XH9Ba5LGXTXWeqZv0ClOD/19a0Z58= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +gioui.org v0.0.0-20210308172011-57750fc8a0a6 h1:K72hopUosKG3ntOPNG4OzzbuhxGuVf06fa2la1/H/Ho= +gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= +github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= +github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= +github.com/Azure/azure-sdk-for-go v16.2.1+incompatible h1:KnPIugL51v3N3WwvaSmZbxukD1WuWXOiE9fRdu32f2I= +github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-storage-blob-go v0.14.0 h1:1BCg74AmVdYwO3dlKwtFU1V0wU2PZdREkXvAmZJRUlM= +github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.1 h1:eVvIXUKiTgv++6YnWb42DUA1YL7qDugnKP0HljexdnQ= +github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/adal v0.9.16 h1:P8An8Z9rH1ldbOLdFpxYorgOt2sywL9V24dAwWHPuGc= +github.com/Azure/go-autorest/autorest/adal v0.9.16/go.mod h1:tGMin8I49Yij6AQ+rvV+Xa/zwxYQB5hmsd6DkfAx2+A= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6nK2Q= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/ClickHouse/clickhouse-go v1.4.3 h1:iAFMa2UrQdR5bHJ2/yaSLffZkxpcOYQMCUuKeNXGdqc= +github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= +github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.5.0 h1:Elr9Wn+sGKPlkaBvwu4mTrxtmOp3F3yV9qhaHbXGjwU= +github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= +github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= +github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= +github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= +github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= +github.com/Microsoft/hcsshim v0.8.21 h1:btRfUDThBE5IKcvI8O8jOiIkujUsAMBSRsYDYmEi6oM= +github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= +github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= +github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3 h1:4FA+QBaydEHlwxg0lMN3rhwoDaQy6LKhVWR4qvq4BuA= +github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46 h1:lsxEuwrXEAokXB9qhlbKWPpo3KMLZQ5WB5WLQRW1uq0= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/Sirupsen/logrus v1.0.6 h1:HCAGQRk48dRVPA5Y+Yh0qdCSTzPOyU1tBJ7Q9YzotII= -github.com/Sirupsen/logrus v1.0.6/go.mod h1:rmk17hk6i8ZSAJkSDa7nOxamrG+SP4P0mm+DAvExv4U= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs= +github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af h1:wVe6/Ea46ZMeNkQjjBW6xcqyQA/j5e0D6GytH95g0gQ= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae h1:AMzIhMUqU3jMrZiTuW0zkYeKlKDAFD+DG20IoO421/Y= +github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= +github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/apache/arrow/go/arrow v0.0.0-20210818145353-234c94e4ce64/go.mod h1:2qMFB56yOP3KzkB3PbYZ4AlUFg3a88F67TIx5lB/WwY= +github.com/apache/arrow/go/arrow v0.0.0-20211013220434-5962184e7a30 h1:HGREIyk0QRPt70R69Gm1JFHDgoiyYpCyuGE8E9k/nf0= +github.com/apache/arrow/go/arrow v0.0.0-20211013220434-5962184e7a30/go.mod h1:Q7yQnSMnLvcXlZ8RV+jwz/6y1rQTqbX6C82SndT52Zs= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 h1:G1bPvciwNyF7IUmKXNt9Ak3m6u9DE1rF+RmtIkBpVdA= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/aws/aws-sdk-go v1.15.34 h1:P0dzidrIxx2/XchvjqoQ/CXIzkCHWkGDa9CLPqzKG2o= -github.com/aws/aws-sdk-go v1.15.34/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= +github.com/aws/aws-sdk-go v1.17.7 h1:/4+rDPe0W95KBmNGYCG+NUvdL8ssPYBMxL+aSCg6nIA= +github.com/aws/aws-sdk-go v1.17.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go-v2 v1.8.0/go.mod h1:xEFuWz+3TYdlPRuo+CqATbeDWIWyaT5uAPwPaWtgse0= +github.com/aws/aws-sdk-go-v2 v1.9.2 h1:dUFQcMNZMLON4BOe273pl0filK9RqyQMhCK/6xssL6s= +github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= +github.com/aws/aws-sdk-go-v2/config v1.6.0/go.mod h1:TNtBVmka80lRPk5+S9ZqVfFszOQAGJJ9KbT3EM3CHNU= +github.com/aws/aws-sdk-go-v2/config v1.8.3 h1:o5583X4qUfuRrOGOgmOcDgvr5gJVSu57NK08cWAhIDk= +github.com/aws/aws-sdk-go-v2/config v1.8.3/go.mod h1:4AEiLtAb8kLs7vgw2ZV3p2VZ1+hBavOc84hqxVNpCyw= +github.com/aws/aws-sdk-go-v2/credentials v1.3.2/go.mod h1:PACKuTJdt6AlXvEq8rFI4eDmoqDFC5DpVKQbWysaDgM= +github.com/aws/aws-sdk-go-v2/credentials v1.4.3 h1:LTdD5QhK073MpElh9umLLP97wxphkgVC/OjQaEbBwZA= +github.com/aws/aws-sdk-go-v2/credentials v1.4.3/go.mod h1:FNNC6nQZQUuyhq5aE5c7ata8o9e4ECGmS4lAXC7o1mQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.4.0/go.mod h1:Mj/U8OpDbcVcoctrYwA2bak8k/HFPdcLzI/vaiXMwuM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.6.0 h1:9tfxW/icbSu98C2pcNynm5jmDwU3/741F11688B6QnU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.6.0/go.mod h1:gqlclDEZp4aqJOancXK6TN24aKhT0W0Ae9MHk3wzTMM= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.4.0/go.mod h1:eHwXu2+uE/T6gpnYWwBwqoeqRf9IXyCcolyOWDRAErQ= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.4 h1:TnU1cY51027j/MQeFy7DIgk1UuzJY+wLFYqXceY/fiE= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.4/go.mod h1:Ex7XQmbFmgFHrjUX6TN3mApKW5Hglyga+F7wZHTtYhA= +github.com/aws/aws-sdk-go-v2/internal/ini v1.2.0/go.mod h1:Q5jATQc+f1MfZp3PDMhn6ry18hGvE0i8yvbXoKbnZaE= +github.com/aws/aws-sdk-go-v2/internal/ini v1.2.4 h1:leSJ6vCqtPpTmBIgE7044B1wql1E4n//McF+mEgNrYg= +github.com/aws/aws-sdk-go-v2/internal/ini v1.2.4/go.mod h1:ZcBrrI3zBKlhGFNYWvju0I3TR93I7YIgAfy82Fh4lcQ= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.2/go.mod h1:EASdTcM1lGhUe1/p4gkojHwlGJkeoRjjr1sRCzup3Is= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.3.0 h1:gceOysEWNNwLd6cki65IMBZ4WAM0MwgBQq2n7kejoT8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.3.0/go.mod h1:v8ygadNyATSm6elwJ/4gzJwcFhri9RqS8skgHKiwXPU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.2/go.mod h1:NXmNI41bdEsJMrD0v9rUvbGCB5GwdBEpKvUvIY3vTFg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.2 h1:r7jel2aa4d9Duys7wEmWqDd5ebpC9w6Kxu6wIjjp18E= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.2/go.mod h1:72HRZDLMtmVQiLG2tLfQcaWLCssELvGl+Zf2WVxMmR8= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.2/go.mod h1:QuL2Ym8BkrLmN4lUofXYq6000/i5jPjosCNK//t6gak= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.7.2 h1:RnZjLgtCGLsF2xYYksy0yrx6xPvKG9BYv29VfK4p/J8= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.7.2/go.mod h1:np7TMuJNT83O0oDOSF8i4dF3dvGqA6hPYYo6YYkzgRA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.12.0/go.mod h1:6J++A5xpo7QDsIeSqPK4UHqMSyPOCopa+zKtqAMhqVQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.16.1 h1:z+P3r4LrwdudLKBoEVWxIORrk4sVg4/iqpG3+CS53AY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.16.1/go.mod h1:CQe/KvWV1AqRc65KqeJjrLzr5X2ijnFTTVzJW0VBRCI= +github.com/aws/aws-sdk-go-v2/service/sso v1.3.2/go.mod h1:J21I6kF+d/6XHVk7kp/cx9YVD2TMD2TbLwtRGVcinXo= +github.com/aws/aws-sdk-go-v2/service/sso v1.4.2 h1:pZwkxZbspdqRGzddDB92bkZBoB7lg85sMRE7OqdB3V0= +github.com/aws/aws-sdk-go-v2/service/sso v1.4.2/go.mod h1:NBvT9R1MEF+Ud6ApJKM0G+IkPchKS7p7c2YPKwHmBOk= +github.com/aws/aws-sdk-go-v2/service/sts v1.6.1/go.mod h1:hLZ/AnkIKHLuPGjEiyghNEdvJ2PP0MgOxcmv9EBJ4xs= +github.com/aws/aws-sdk-go-v2/service/sts v1.7.2 h1:ol2Y5DWqnJeKqNd8th7JWzBtqu63xpOfs1Is+n1t8/4= +github.com/aws/aws-sdk-go-v2/service/sts v1.7.2/go.mod h1:8EzeIqfWt2wWT4rJVu3f21TfrhJ8AEMzVybRNSb/b4g= +github.com/aws/smithy-go v1.7.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/aws/smithy-go v1.8.0 h1:AEwwwXQZtUwP5Mz506FeXXrKBe0jA8gVM+1gEcSRooc= +github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= +github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= +github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= +github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/boombuler/barcode v1.0.0 h1:s1TvRnXwL2xJRaccrdcBQMZxq6X7DvsMogtmJeHDdrc= +github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/bshuster-repo/logrus-logstash-hook v0.4.1 h1:pgAtgj+A31JBVtEHu2uHuEx0n+2ukqUJnS2vVe5pQNA= +github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= +github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44 h1:y853v6rXx+zefEcjET3JuKAqvhj+FKflQijjeaSv2iA= +github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd h1:rFt+Y/IK1aEZkEHchZRSq9OQbsSzIT/OrI8YFFmRIng= +github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= +github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembjv71DPz3uX/V/6MMlSyD9JBQ6kQ= +github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= +github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 h1:nvj0OLI3YqYXer/kZD8Ri1aaunCxIEsOst1BVJswV0o= +github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs= +github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= +github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/checkpoint-restore/go-criu/v4 v4.1.0 h1:WW2B2uxx9KWF6bGlHqhm8Okiafwwx7Y2kcpn8lCpjgo= +github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= +github.com/checkpoint-restore/go-criu/v5 v5.0.0 h1:TW8f/UvntYoVDMN1K2HlT82qH1rb0sOjpGw3m6Ym+i4= +github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= +github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= +github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc= +github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= +github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= +github.com/cilium/ebpf v0.6.2 h1:iHsfF/t4aW4heW2YKfeHrVPGdtYTL4C4KocpM8KTSnI= +github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= +github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cockroachdb/cockroach-go v0.0.0-20180212155653-59c0560478b7 h1:XFqp7VFIbbJO1hlpGbzo45NVYWVIM2eMD9MAxrOTVzU= -github.com/cockroachdb/cockroach-go v0.0.0-20180212155653-59c0560478b7/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403 h1:cqQfy1jclcSy/FwLjemeg3SR1yaINm74aQyupQ0Bl8M= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158 h1:CevA8fI91PAnP8vpnXuB8ZYAZ5wqY86nAbxfgK8tWO4= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/cockroachdb/cockroach-go/v2 v2.1.1 h1:3XzfSMuUT0wBe1a3o5C0eOTcArhmmFAg2Jzh/7hhKqo= +github.com/cockroachdb/cockroach-go/v2 v2.1.1/go.mod h1:7NtUnP6eK+l6k483WSYNrq3Kb23bWV10IRV1TyeSpwM= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= +github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= +github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= +github.com/containerd/aufs v1.0.0 h1:2oeJiwX5HstO7shSrPZjrohJZLzK36wvpdmzDRkL/LY= +github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= +github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E= +github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= +github.com/containerd/btrfs v1.0.0 h1:osn1exbzdub9L5SouXO5swW4ea/xVdJZ3wokxN5GrnA= +github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= +github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI= +github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= +github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= +github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= +github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= +github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= +github.com/containerd/cgroups v1.0.1 h1:iJnMvco9XGvKUvNQkv88bE4uJXxRQH18efbKo9w5vHQ= +github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= +github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= +github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= +github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= +github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= +github.com/containerd/console v1.0.2 h1:Pi6D+aZXM+oUw1czuKgH5IJ+y0jhYcwBJfx5/Ghn9dE= +github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= +github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ= +github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= +github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= +github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= +github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= +github.com/containerd/containerd v1.5.7 h1:rQyoYtj4KddB3bxG6SAqd4+08gePNyJjRqvOIfV3rkM= +github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= +github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo= +github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= +github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= +github.com/containerd/continuity v0.1.0 h1:UFRRY5JemiAhPZrr/uE0n8fMTLcZsUvySPr1+D7pgr8= +github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= +github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= +github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= +github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= +github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= +github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= +github.com/containerd/fifo v1.0.0 h1:6PirWBr9/L7GDamKr+XM0IeUFXu5mf3M/BPpH9gaLBU= +github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= +github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= +github.com/containerd/go-cni v1.0.2 h1:YbJAhpTevL2v6u8JC1NhCYRwf+3Vzxcc5vGnYoJ7VeE= +github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk= +github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= +github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= +github.com/containerd/go-runc v1.0.0 h1:oU+lLv1ULm5taqgV/CJivypVODI4SUz1znWjv3nNYS0= +github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= +github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0= +github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= +github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= +github.com/containerd/imgcrypt v1.1.1 h1:LBwiTfoUsdiEGAR1TpvxE+Gzt7469oVu87iR3mv3Byc= +github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms= +github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= +github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= +github.com/containerd/nri v0.1.0 h1:6QioHRlThlKh2RkRTR4kIT3PKAcrLo3gIWnjkM4dQmQ= +github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= +github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= +github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= +github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= +github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= +github.com/containerd/ttrpc v1.0.2 h1:2/O3oTZN36q2xRolk0a2WWGgh7/Vf/liElg5hFYLX9U= +github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= +github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= +github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= +github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= +github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY= +github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= +github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw= +github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y= +github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containerd/zfs v0.0.0-20210324211415-d5c4544f0433/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containerd/zfs v1.0.0 h1:cXLJbx+4Jj7rNsTiqVfm6i+RNLx6FFA2fMmDlEf+Wm8= +github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v0.8.1 h1:7zpDnQ3T3s4ucOuJ/ZCLrYBxzkg0AELFfII3Epo9TmI= +github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= +github.com/containernetworking/plugins v0.9.1 h1:FD1tADPls2EEi3flPc2OegIY1M9pUa9r2Quag7HMLV8= +github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= +github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= +github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= +github.com/containers/ocicrypt v1.1.1 h1:prL8l9w3ntVqXvNH1CiNn5ENjcCnr38JqpSyvKKB4GI= +github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= +github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ7pD7MnhC04= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= +github.com/coreos/go-iptables v0.5.0 h1:mw6SAibtHKZcNzAsOxjoHIG0gy5YFHhypWSSNc6EjbQ= +github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= +github.com/coreos/go-oidc v2.1.0+incompatible h1:sdJrfw8akMnCuUlaZU3tE/uYXFgfqom8DBE9so9EBsM= +github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07 h1:UHFGPvSxX4C4YBApSPvmUfL8tTvWLj2ryqvT9K4Jcuk= -github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= -github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f h1:7uSNgsgcarNk4oiN/nNkO0J7KAjlsF5Yv5Gf/tFdHas= -github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f/go.mod h1:8S58EK26zhXSxzv7NQFpnliaOQsmDUxvoQO3rt154Vg= -github.com/cznic/golex v0.0.0-20170803123110-4ab7c5e190e4 h1:CVAqftqbj+exlab+8KJQrE+kNIVlQfJt58j4GxCMF1s= -github.com/cznic/golex v0.0.0-20170803123110-4ab7c5e190e4/go.mod h1:+bmmJDNmKlhWNG+gwWCkaBoTy39Fs+bzRxVBzoTQbIc= -github.com/cznic/internal v0.0.0-20180608152220-f44710a21d00 h1:FHpbUtp2K8X53/b4aFNj4my5n+i3x+CQCZWNuHWH/+E= -github.com/cznic/internal v0.0.0-20180608152220-f44710a21d00/go.mod h1:olo7eAdKwJdXxb55TKGLiJ6xt1H0/tiiRCWKVLmtjY4= -github.com/cznic/lldb v1.1.0 h1:AIA+ham6TSJ+XkMe8imQ/g8KPzMUVWAwqUQQdtuMsHs= -github.com/cznic/lldb v1.1.0/go.mod h1:FIZVUmYUVhPwRiPzL8nD/mpFcJ/G7SSXjjXYG4uRI3A= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg= +github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369 h1:XNT/Zf5l++1Pyg08/HV04ppB0gKxAqtZQBRYiYrUuYk= github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= -github.com/cznic/ql v1.2.0 h1:lcKp95ZtdF0XkWhGnVIXGF8dVD2X+ClS08tglKtf+ak= -github.com/cznic/ql v1.2.0/go.mod h1:FbpzhyZrqr0PVlK6ury+PoW3T0ODUV22OeWIxcaOrSE= -github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 h1:hxuZop6tSoOi0sxFzoGGYdRqNrPubyaIf9KoBG9tPiE= -github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= -github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186 h1:0rkFMAbn5KBKNpJyHQ6Prb95vIKanmAe62KxsrN+sqA= -github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= -github.com/cznic/zappy v0.0.0-20160723133515-2533cb5b45cc h1:YKKpTb2BrXN2GYyGaygIdis1vXbE7SSAG9axGWIMClg= -github.com/cznic/zappy v0.0.0-20160723133515-2533cb5b45cc/go.mod h1:Y1SNZ4dRUOKXshKUbwUapqNncRrho4mkjQebgEHZLj8= +github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c h1:Xo2rK1pzOm0jO6abTPIQwbAmqBIOj132otexc1mmzFc= +github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= +github.com/d2g/dhcp4client v1.0.0 h1:suYBsYZIkSlUMEz4TAYCczKf62IA2UWC+O8+KtdOhCo= +github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= +github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5 h1:+CpLbZIeUn94m02LdEKPcgErLJ347NUwxPKs5u8ieiY= +github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= +github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4 h1:itqmmf1PFpC4n5JW+j4BU7X4MTfVurhYRTjODoPb2Y8= +github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.10.0 h1:QykgLZBorFE95+gO3u9esLd0BmbvpWp0/waNNZfHBM8= +github.com/denisenkom/go-mssqldb v0.10.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba h1:p6poVbjHDkKa+wtC8frBMwQtT3BmqGYBjzMwJ63tuR4= +github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= +github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954 h1:RMLoZVzv4GliuWafOuPuQDKSm1SJph7uCRnnS61JAn4= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docker/distribution v0.0.0-20180720172123-0dae0957e5fe h1:ZRQNMB7Sw5jf9g/0imDbI+vTFNk4J7qBdtFI5/zf1kg= -github.com/docker/distribution v0.0.0-20180720172123-0dae0957e5fe/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v0.0.0-20170502054910-90d35abf7b35 h1:ly3dRUfvdP5i/t9iqVHd2VQQIDtO3tpfFWPah7g4CFw= -github.com/docker/docker v0.0.0-20170502054910-90d35abf7b35/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/dhui/dktest v0.3.7 h1:jWjWgHAPDAdqgUr7lAsB3bqB2DKWC3OaA+isfekjRew= +github.com/dhui/dktest v0.3.7/go.mod h1:nYMOkafiA07WchSwKnKFUSbGMb2hMm5DrCGiXYG6gwM= +github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= +github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= +github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v20.10.9+incompatible h1:JlsVnETOjM2RLQa0Cc1XCIspUdXW3Zenq9P54uXBm6k= +github.com/docker/docker v20.10.9+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= -github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= -github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= +github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= +github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= +github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4= +github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh6AyO7hdCn/PkvCZXii8TGj7sbtEbQ= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712 h1:aaQcKT9WumO6JEJcRyTqFVq4XUZiUcKR2/GI31TOcz8= github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021 h1:fP+fF0up6oPY49OrjPrhIJ8yQfdIM85NXMLkMg1EXVs= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8= +github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.5+incompatible h1:/l4kBbb4/vGSsdtB5nUe8L7B9mImVMaBPw9L/0TBHU8= +github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsouza/fake-gcs-server v1.2.0 h1:FZUL/EJlyAlHxpUWZs23ae4zNwBwmHM1p5TykkoP85A= -github.com/fsouza/fake-gcs-server v1.2.0/go.mod h1:rM69NBSmfAkTlKDhzXC41OeX9lQxQXnkimkiGWDU1Ek= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsouza/fake-gcs-server v1.17.0 h1:OeH75kBZcZa3ZE+zz/mFdJ2btt9FgqfjI7gIh9+5fvk= +github.com/fsouza/fake-gcs-server v1.17.0/go.mod h1:D1rTE4YCyHFNa99oyJJ5HyclvN/0uQR+pM/VdlL83bw= +github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa h1:RDBNVkRviHZtvDvId8XSGPu3rmpmSe+wKRcEWNgsfWU= +github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= +github.com/gabriel-vasile/mimetype v1.3.1/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8= +github.com/gabriel-vasile/mimetype v1.4.0 h1:Cn9dkdYsMIu56tGho+fqzh7XmvY2YyGU0FnbhiOsEro= +github.com/gabriel-vasile/mimetype v1.4.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8= +github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7 h1:LofdAjjjqCSXMwLGgOgnE+rdPuvX9DxCqaHwKy7i/ko= +github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-fonts/dejavu v0.1.0 h1:JSajPXURYqpr+Cu8U9bt8K+XcACIHWqWrvWCKyeFmVQ= +github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= +github.com/go-fonts/latin-modern v0.2.0 h1:5/Tv1Ek/QCr20C6ZOz15vw3g7GELYL98KWr8Hgo+3vk= +github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= +github.com/go-fonts/liberation v0.1.1 h1:wBrPaMkrXFBW3qXpXAjiKljdVUMxn9bX2ia3XjPHoik= +github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/stix v0.1.0 h1:UlZlgrvvmT/58o573ot7NFw0vZasZ5I6bcIft/oMdgg= +github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ini/ini v1.25.4 h1:Mujh4R/dH6YL8bxuISne3xX2+qcQ9p0IxKAP6ExWoUo= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-ini/ini v1.38.2 h1:6Hl/z3p3iFkA0dlDfzYxuFuUGD+kaweypF6btsR2/Q4= -github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07 h1:OTlfMvwR1rLyf9goVmXfuS5AJn80+Vmj4rTf4n46SOs= +github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gocql/gocql v0.0.0-20180913072538-864d5908455a h1:GweitK1CmWn9k5Y39d3zhQ3HXTI4KaDSXSlWUQ6qQEw= -github.com/gocql/gocql v0.0.0-20180913072538-864d5908455a/go.mod h1:4Fw1eo5iaEhDUs8XyuhSVCVy52Jq3L+/3GJgYkwc+/0= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd h1:hSkbZ9XSyjyBirMeqSqUrK+9HboWrweVlzRNqoBi2d4= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= +github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/depgen v0.1.0 h1:31atYa/UW9V5q8vMJ+W6wd64OaaTHUrCUXER358zLM4= +github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= +github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.7.0 h1:GlXgaiBkmrYMHco6t4j7SacKO4XUjvh5pwXh0f4uxXU= +github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= +github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.1.3 h1:3GQ53z7E3o00C/yy7Ko8VXqQXoJGLkrTQCLTF1EjoXU= +github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= +github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= +github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= +github.com/gobuffalo/genny v0.1.1 h1:iQ0D6SpNXIxu52WESsD+KoQ7af2e3nCfnSBoSF/hKe0= +github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211 h1:mSVZ4vj4khv+oThUfS+SQU3UuFIZ5Zo6UNcvK8E8Mz8= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= +github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= +github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= +github.com/gobuffalo/gogen v0.1.1 h1:dLg+zb+uOyd/mKeQUYIbwbNmfRsr9hd/WtYWepmayhI= +github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= +github.com/gobuffalo/here v0.6.0 h1:hYrd0a6gDmWxBM4TnrGw8mQg24iSVoIkHEk7FodQcBI= +github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2 h1:8thhT+kUJMTMy3HlX4+y9Da+BNJck+p109tqqKp7WDs= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= +github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2 h1:fq9WcL1BYrm36SzK6+aAnZ8hcp+SrmnDyAxhNx8dvJk= +github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.1.0 h1:4sGKOD8yaYJ+dek1FDkwcxCHA40M4kfKgFHx8N2kwbU= +github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= +github.com/gobuffalo/packr/v2 v2.2.0 h1:Ir9W9XIm9j7bhhkKE9cokvtTl1vBm62A/fene/ZCj6A= +github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754 h1:tpom+2CJmpzAWj5/VEHync2rJGi+epHNIeRSWjzGA+4= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/gocql/gocql v0.0.0-20210515062232-b7ef815b4556 h1:N/MD/sr6o61X+iZBAT2qEUF023s4KbA8RWfKzl0L6MQ= +github.com/gocql/gocql v0.0.0-20210515062232-b7ef815b4556/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY= +github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e h1:BWhy2j3IXJhjCbC68FptL43tDKIq8FladmaTs3Xs7Z8= +github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= +github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= +github.com/gogo/googleapis v1.4.0 h1:zgVt4UpGxcqVOw97aRGxT4svlcmdK35fynLNctY32zI= +github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.1.0 h1:XUgk2Ex5veyVFVeLm0xhusUTQybEbexJXrvPNOKkSY0= +github.com/golang-jwt/jwt/v4 v4.1.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= +github.com/google/flatbuffers v2.0.0+incompatible h1:dicJ2oXwypfwUGnB2/TYWYEKiuk9eYQlQO/AnOHl5mI= +github.com/google/flatbuffers v2.0.0+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= -github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= -github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 h1:zLTLjkaOFEFIOxY5BWLFLwh+cL8vOBW4XJ2aqLE/Tf0= -github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-github/v39 v39.2.0 h1:rNNM311XtPOz5rDdsJXAp2o8F67X9FnROXTvto3aSnQ= +github.com/google/go-github/v39 v39.2.0/go.mod h1:C1s8C5aCC9L+JXIYpJM5GYytdX52vC1bLvHEF1IhBrE= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/googleapis/gax-go v2.0.0+incompatible h1:j0GKcs05QVmm7yesiZq2+9cxHkNK9YM6zKx4D2qucQU= -github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210715191844-86eeefc3e471 h1:WqmlwDwojb0rrPPtueSYqNrONX90T3SjwZeVUr4QCtI= +github.com/google/pprof v0.0.0-20210715191844-86eeefc3e471/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= -github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YARg= +github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= +github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= +github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 h1:mV02weKRL81bEnm8A0HT1/CAelMQDBuQIfLw8n+d6xI= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56 h1:742eGXur0715JMq73aD95/FU0XpVKXqNuTnEfXsLOYQ= +github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk= +github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.8.0 h1:FmjZ0rOyXTr1wfWs45i4a9vjnjWUAGpMuQLD9OSs+lw= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgerrcode v0.0.0-20201024163028-a0d42d470451 h1:WAvSpGf7MsFuzAtK4Vk7R4EVe+liW4x83r4oWu0WHKw= +github.com/jackc/pgerrcode v0.0.0-20201024163028-a0d42d470451/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2 h1:JVX6jT/XfzNqIjye4717ITLaNwV9mWbJx0dLCpcRzdA= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.7 h1:6Pwi1b3QdY65cuv6SyVO0FgPd5J3Bl7wf/nQQjinHMA= +github.com/jackc/pgproto3/v2 v2.0.7/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0= +github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po= +github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= +github.com/jackc/pgtype v1.6.2 h1:b3pDeuhbbzBYcg5kwNmNDun4pFUD/0AAr1kLXZLeNt8= +github.com/jackc/pgtype v1.6.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA= +github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o= +github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= +github.com/jackc/pgx/v4 v4.10.1 h1:/6Q3ye4myIj6AaplUm+eRcz4OhK9HAvFf4ePsG40LJY= +github.com/jackc/pgx/v4 v4.10.1/go.mod h1:QlrWebbs3kqEZPHCTGyxecvzG6tvIsYu+A5b1raylkA= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3 h1:JnPg/5Q9xVJGfjsO5CPUOjnJps1JaRUm8I9FXVCFK94= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E= +github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/jmoiron/sqlx v1.3.1 h1:aLN7YINNZ7cYOPK3QC83dbM6KT0NMqVMw961TqrejlE= +github.com/jmoiron/sqlx v1.3.1/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= +github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5 h1:PJr+ZMXIecYc1Ey2zucXdR73SMBtgjPgwa31099IMv0= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/k0kubun/pp v2.3.0+incompatible h1:EKhKbi34VQDWJtq+zpsKSEhkHHs9w2P8Izbq8IhLVSo= +github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/karrick/godirwalk v1.10.3 h1:lOpSw2vJP0y5eLBW906QwKsUK/fe/QDyoqM5rnnuPDY= +github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kshvakov/clickhouse v1.3.4 h1:p/yqvOmeDRH+KyCH6NtwExelr4rimLBBfKW2a/wBN94= -github.com/kshvakov/clickhouse v1.3.4/go.mod h1:DMzX7FxRymoNkVgizH0DWAL8Cur7wHLgx3MUnGwJqpE= -github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= +github.com/ktrysmt/go-bitbucket v0.6.4 h1:C8dUGp0qkwncKtAnozHCbbqhptefzEd1I0sfnuy9rYQ= +github.com/ktrysmt/go-bitbucket v0.6.4/go.mod h1:9u0v3hsd2rqCHRIpbir1oP7F58uo5dq19sBYvuMoyQ4= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E= +github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2 h1:JgVTCPf0uBVcUSWpyXmGpgOc62nK5HWUBKAGc3Qqa5k= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/pkger v0.15.1 h1:3MPelV53RnGSW07izx5xGxl4e/sdRD6zqseIk0rMASY= +github.com/markbates/pkger v0.15.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI= +github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/marstr/guid v1.1.0 h1:/M4H/1G4avsieL6BbUwCOBzulmoeKVP5ux/3mQNnbyI= +github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= +github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.2 h1:UnlwIPBGaTZfPQ6T1IGzPI0EkYAQmT9fAEJ/poFC63o= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-shellwords v1.0.3 h1:K/VxK7SZ+cvuPgFSLKi5QPI9Vr/ipOf4C1gN+ntueUk= +github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/pkcs11 v1.0.3 h1:iMwmD7I5225wv84WxIG/bmxz9AXjWvTWIbM/TYHvWtw= +github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible h1:aKW/4cBs+yK6gpqU3K/oIwk9Q/XICqd3zOX/UFuvqmk= +github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f h1:2+myh5ml7lgEU/51gbeLHfKGNfgEQQIWrlbdaOsidbQ= +github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= +github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= +github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= +github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/mountinfo v0.4.1 h1:1O+1cHA1aujwEwwVMa2Xm2l+gIpUHyd3+D+d7LZh1kM= +github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/symlink v0.1.0 h1:MTFZ74KtNI6qQQpuBxU+uKCim4WtOMokr03hCfJcazE= +github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= +github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mrunalp/fileutils v0.5.0 h1:NKzVxiH7eSk+OQ4M+ZYW1K6h27RUV3MI6NUTsHhU6Z4= +github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mutecomm/go-sqlcipher/v4 v4.4.0 h1:sV1tWCWGAVlPhNGT95Q+z/txFxuhAYWwHD1afF5bMZg= +github.com/mutecomm/go-sqlcipher/v4 v4.4.0/go.mod h1:PyN04SaWalavxRGH9E8ZftG6Ju7rsPrGmQRjrEaVpiY= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8 h1:P48LjvUQpTReR3TQRbxSeSBsMXzfK0uol7eRcr7VBYQ= +github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8/go.mod h1:86wM1zFnC6/uDBfZGNwB65O+pR2OFi5q/YQaEUid1qA= +github.com/ncw/swift v1.0.47 h1:4DQRPj35Y41WogBxyhOXlrI37nzGlyEcsforeudyYPQ= +github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= +github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba h1:fhFP5RliM2HW/8XdcO5QngSfFli9GcRIpMXvypTQt6E= +github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba/go.mod h1:ncO5VaFWh0Nrt+4KT4mOZboaczBZcLuHrG+/sUeP8gI= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5 h1:58+kh9C6jJVXYjt8IE48G2eWl6BjwU5Gj0gqY84fy78= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I= -github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= +github.com/onsi/ginkgo v1.12.1 h1:mFwc4LvZ0xpSvDZ3E+k8Yte0hLOMxXUlP+yXtJqkYfQ= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= +github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA= +github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= +github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= +github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= +github.com/opencontainers/runc v1.0.2 h1:opHZMaswlyxz1OuGpBE53Dwe4/xF7EZTY0A2L/FpCOg= +github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= +github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 h1:3snG66yBm59tKhhSPQrQ/0bCrv1LQbKt40LnUPiUxdc= +github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39 h1:H7DMc6FAjgwZZi8BRqjrAAHWoqEr5e5L6pS4V0ezet4= +github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= +github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= +github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= +github.com/opencontainers/selinux v1.8.2 h1:c4ca10UMgRcvZ6h0K4HtS15UaVSBEaE+iln2LVpAuGc= +github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= +github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/phpdave11/gofpdf v1.4.2 h1:KPKiIbfwbvC/wOncwhrpRdXVj2CZTCFlw4wnoyjtHfQ= +github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= +github.com/phpdave11/gofpdi v1.0.12 h1:RZb9NG62cw/RW0rHAduVRo+98R8o/G1krcg2ns7DakQ= +github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.8 h1:ieHkV+i2BRzngO4Wd/3HGowuZStgq6QkPsD1eolNAO4= +github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/browser v0.0.0-20210706143420-7d21f8c997e2/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021 h1:0XM1XL/OFFJjXsYXlG30spTkV/E9+gmd5GD1w2HE8xM= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= +github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= +github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0 h1:uPRuwkWF4J6fGsJ2R0Gn2jB1EQiav9k3S6CSdygQJXY= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58 h1:nlG4Wa5+minh3S9LVFtNoY+GVRiudA2e3EVfcCi3RCA= +github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= +github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8 h1:2c1EFnZHIPCW8qKWgHMH/fX2PkSabFc5mrVzfUNdg5U= +github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/seccomp/libseccomp-golang v0.9.1 h1:NJjM5DNFOs0s3kYE1WUOr6G8V97sdt46rlXTMfXGWBo= +github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/snowflakedb/gosnowflake v1.6.3 h1:EJDdDi74YbYt1ty164ge3fMZ0eVZ6KA7b1zmAa/wnRo= +github.com/snowflakedb/gosnowflake v1.6.3/go.mod h1:6hLajn6yxuJ4xUHZegMekpq9rnQbGJ7TMwXjgTmA6lg= +github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E= -github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= +github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 h1:lIOOHPEbXzO3vnmx2gok1Tfs31Q8GQqKLc8vVqyQq/I= +github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= +github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/tchap/go-patricia v2.2.6+incompatible h1:JvoDL7JSoIP2HDE8AbDH3zC8QBPxmzYe32HHy5yQ+Ck= +github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo= +github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 h1:cPXZWzzG0NllBLdjWoD1nDfaqu98YMv+OneaKc8sPOA= +github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= +github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns= +github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE= +github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= +github.com/xanzy/go-gitlab v0.15.0 h1:rWtwKTgEnXyNUGrOArN7yyc3THRkpYcKXIXia9abywQ= +github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.0.2 h1:akYIkZ28e6A96dkWNJQu3nmCzH3YfwMPQExUYDaRv7w= +github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= +github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyhBc= +github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f h1:mvXjJIHRZyhNuGassLTcXTwjiWq7NmjdavZsUnmFybQ= +github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77 h1:ESFSdwYZvkeru3RtdrYueztKhOBCSAAzS4Gf+k0tEow= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43 h1:+lm10QQTNSBd8DVTNGHx7o/IKu9HYDvLMffDhbyLccI= +github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= +github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50 h1:hlE8//ciYMztlGpl/VA+Zm1AcTPHYkHJPbHqE6WJUXE= +github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= +github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f h1:ERexzlUfuTvpE74urLSbIQW0Z/6hF9t8U4NsJLaioAY= +github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= +github.com/zenazn/goji v0.9.0 h1:RSQQAbXGArQ0dIDEq+PI6WqN6if+5KHu6x2Cx/GXLTQ= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b h1:7gd+rd8P3bqcn/96gOZa3F5dpJr/vEiDQYlNb/y2uNs= +gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0= -go.opencensus.io v0.16.0 h1:AAKPvhQpIeldLYe+sBlzGOU331MfHxWtPLkTF/t2oYs= -go.opencensus.io v0.16.0/go.mod h1:0TeCCqcQSLNZtiq/62+vUzqwnjqF5el6hjmuZaFtyNk= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489 h1:1JFLBqwIgdyHN1ZtgjTBwO+blA6gVOmZurpiMEsETKo= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= +go.mongodb.org/mongo-driver v1.7.0 h1:hHrvOBWlWB2c7+8Gh/Xi5jj82AgidK/t7KVXBZ+IyUA= +go.mongodb.org/mongo-driver v1.7.0/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8= +go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1 h1:A/5uWzF44DlIgdm/PQFwfMkW0JX+cIcQi/SwLAmZP5M= +go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/proto/otlp v0.7.0 h1:rwOQPCuKAKmwGKq2aVNnYIibI6wnV7EvzgfTCzcdGg8= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= +golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210216034530-4410531fe030 h1:lP9pYkih3DUSC641giIXa2XqfTIbbbRr0w2EOTA7wHA= +golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/net v0.0.0-20180821023952-922f4815f713/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190225153610-fe579d43d832/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211013171255-e13a2654a71e h1:Xj+JO91noE97IN6F/7WZxzC5QE6yENAQPrwIYhW3bsA= +golang.org/x/net v0.0.0-20211013171255-e13a2654a71e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/oauth2 v0.0.0-20180227000427-d7d64896b5ff/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914 h1:3B43BWw0xEBsLZ/NO1VALz6fppU3481pik+2Ksv45z8= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180821140842-3b58ed4ad339/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180224232135-f6cff0780e54/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210818153620-00dd8d7831e7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211013075003-97ac67df715c h1:taxlMj0D/1sOAuv/CbSD+MMDof2vbyPTqz5FNYKpXt8= +golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 h1:TFlARGu6Czu1z7q93HTxcP1P+/ZFC/IKythI5RzrnRg= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -google.golang.org/api v0.0.0-20180818000503-e21acd801f91/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/api v0.0.0-20180826000528-7954115fcf34/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf h1:rjxqQmxjyqerRKEj+tZW+MCm4LgpFXu18bsEoCMgDsk= -google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs= +golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/gonum v0.9.3 h1:DnoIG+QAMaF5NvxnGe/oKsgKcAc6PcUyl8q0VetfQ8s= +gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +gonum.org/v1/plot v0.9.0 h1:3sEo36Uopv1/SA/dMFFaxXoL5XyikJ9Sf2Vll/k6+2E= +gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= +google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0 h1:SQaA2Cx57B+iPw2MBgyjEkoeMkRK2IenSGoia0U3lCk= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/appengine v1.0.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8 h1:Cpp2P6TPjujNoC5M2KHY6g7wfyLYfIWRZaSdIKfDasA= +google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20180912233945-5a2fd4cab2d6 h1:YN6g8chEdBTmaERxJzbJ6WKLrW3+Bf6rznOBkWNSQP0= -google.golang.org/genproto v0.0.0-20180912233945-5a2fd4cab2d6/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210207032614-bba0dbe2a9ea/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210630183607-d20f26d13c79/go.mod h1:yiaVoXHpRzHGyxV3o4DktVWY4mSUErTKaeEOq6C3t3U= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210721163202-f1cecdd8b78a/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210726143408-b02e89920bf0/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20211013025323-ce878158c4d4 h1:NBxB1XxiWpGqkPUiJ9PoBXkHV5A9+GohMOA+EmWoPbU= +google.golang.org/genproto v0.0.0-20211013025323-ce878158c4d4/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.21.0 h1:G+97AoqBnmZIT91cLG/EkCoK9NSelj64P8bOHHNmGn0= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.41.0 h1:f+PlOh7QV4iIJkPrx5NQ7qaNGFQ3OTse67yaDHfju4E= +google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/cheggaaa/pb.v1 v1.0.25 h1:Ev7yu1/f6+d+b3pi5vPdRPc6nNtP1umSfcWiEfRqv6I= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec h1:RlWgLqCMMIYYEVcAR5MDsuHlVkaIPDAF+5Dehzg8L5A= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0 h1:CuXP0Pjfw9rOuY6EP+UvtNvt5DSqHpIxILZKT/quCZI= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.5.1 h1:7odma5RETjNHWJnR32wx8t+Io4djHE1PqxCFx3iiZ2w= +gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.0.8 h1:PAgM+PaHOSAeroTjHkCHCBIHHoBIf9RgPWGo8dF2DA8= +gorm.io/driver/postgres v1.0.8/go.mod h1:4eOzrI1MUfm6ObJU/UcmbXyiHSs8jSwH95G5P5dxcAg= +gorm.io/gorm v1.20.12/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gorm.io/gorm v1.21.4 h1:J0xfPJMRfHgpVcYLrEAIqY/apdvTIkrltPQNHQLq9Qc= +gorm.io/gorm v1.21.4/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= +k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= +k8s.io/api v0.20.6 h1:bgdZrW++LqgrLikWYNruIKAtltXbSCX2l5mJu11hrVE= +k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= +k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.20.6 h1:R5p3SlhaABYShQSO6LpPsYHjV05Q+79eBUR0Ut/f4tk= +k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= +k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= +k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= +k8s.io/apiserver v0.20.6 h1:NnVriMMOpqQX+dshbDoZixqmBhfgrPk2uOh2fzp9vHE= +k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= +k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= +k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= +k8s.io/client-go v0.20.6 h1:nJZOfolnsVtDtbGJNCxzOtKUAu7zvXjB8+pMo9UNxZo= +k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= +k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= +k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= +k8s.io/component-base v0.20.6 h1:G0inASS5vAqCpzs7M4Sp9dv9d0aElpz39zDHbSB4f4g= +k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= +k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= +k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= +k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= +k8s.io/cri-api v0.20.6 h1:iXX0K2pRrbR8yXbZtDK/bSnmg/uSqIFiVJK1x4LUOMc= +k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac h1:sAvhNk5RRuc6FNYGqe7Ygz3PSo/2wGWbulskmzRX8Vs= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= +k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/kubernetes v1.13.0 h1:qTfB+u5M92k2fCCCVP2iuhgwwSOv1EkAkvQY1tQODD8= +k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +modernc.org/b v1.0.0 h1:vpvqeyp17ddcQWF29Czawql4lDdABCDRbXRAS4+aF2o= +modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg= +modernc.org/cc/v3 v3.32.4 h1:1ScT6MCQRWwvwVdERhGPsPq0f55J1/pFEOCiqM7zc78= +modernc.org/cc/v3 v3.32.4/go.mod h1:0R6jl1aZlIl2avnYfbfHBS1QB6/f+16mihBObaBC878= +modernc.org/ccgo/v3 v3.9.2 h1:mOLFgduk60HFuPmxSix3AluTEh7zhozkby+e1VDo/ro= +modernc.org/ccgo/v3 v3.9.2/go.mod h1:gnJpy6NIVqkETT+L5zPsQFj7L2kkhfPMzOghRNv/CFo= +modernc.org/db v1.0.0 h1:2c6NdCfaLnshSvY7OU09cyAY0gYXUZj4lmg5ItHyucg= +modernc.org/db v1.0.0/go.mod h1:kYD/cO29L/29RM0hXYl4i3+Q5VojL31kTUVpVJDw0s8= +modernc.org/file v1.0.0 h1:9/PdvjVxd5+LcWUQIfapAWRGOkDLK90rloa8s/au06A= +modernc.org/file v1.0.0/go.mod h1:uqEokAEn1u6e+J45e54dsEA/pw4o7zLrA2GwyntZzjw= +modernc.org/fileutil v1.0.0 h1:Z1AFLZwl6BO8A5NldQg/xTSjGLetp+1Ubvl4alfGx8w= +modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= +modernc.org/golex v1.0.0 h1:wWpDlbK8ejRfSyi0frMyhilD3JBvtcx2AdGDnU+JtsE= +modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= +modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= +modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= +modernc.org/internal v1.0.0 h1:XMDsFDcBDsibbBnHB2xzljZ+B1yrOVLEFkKL2u15Glw= +modernc.org/internal v1.0.0/go.mod h1:VUD/+JAkhCpvkUitlEOnhpVxCgsBI90oTzSCRcqQVSM= +modernc.org/libc v1.7.13-0.20210308123627-12f642a52bb8/go.mod h1:U1eq8YWr/Kc1RWCMFUWEdkTg8OTcfLw2kY8EDwl039w= +modernc.org/libc v1.9.5 h1:zv111ldxmP7DJ5mOIqzRbza7ZDl3kh4ncKfASB2jIYY= +modernc.org/libc v1.9.5/go.mod h1:U1eq8YWr/Kc1RWCMFUWEdkTg8OTcfLw2kY8EDwl039w= +modernc.org/lldb v1.0.0 h1:6vjDJxQEfhlOLwl4bhpwIz00uyFK4EmSYcbwqwbynsc= +modernc.org/lldb v1.0.0/go.mod h1:jcRvJGWfCGodDZz8BPwiKMJxGJngQ/5DrRapkQnLob8= +modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= +modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.2.2 h1:+yFk8hBprV+4c0U9GjFtL+dV3N8hOJ8JCituQcMShFY= +modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.0.4 h1:utMBrFcpnQDdNsmM6asmyH/FM9TqLPS7XF7otpJmrwM= +modernc.org/memory v1.0.4/go.mod h1:nV2OApxradM3/OVbs2/0OsP6nPfakXpi50C7dcoHXlc= +modernc.org/opt v0.1.1 h1:/0RX92k9vwVeDXj+Xn23DKp2VJubL7k8qNffND6qn3A= +modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/ql v1.0.0 h1:bIQ/trWNVjQPlinI6jdOQsi195SIturGo3mp5hsDqVU= +modernc.org/ql v1.0.0/go.mod h1:xGVyrLIatPcO2C1JvI/Co8c0sr6y91HKFNy4pt9JXEY= +modernc.org/sortutil v1.1.0 h1:oP3U4uM+NT/qBQcbg/K2iqAX0Nx7B1b6YZtq3Gk/PjM= +modernc.org/sortutil v1.1.0/go.mod h1:ZyL98OQHJgH9IEfN71VsamvJgrtRX9Dj2gX+vH86L1k= +modernc.org/sqlite v1.10.6 h1:iNDTQbULcm0IJAqrzCm2JcCqxaKRS94rJ5/clBMRmc8= +modernc.org/sqlite v1.10.6/go.mod h1:Z9FEjUtZP4qFEg6/SiADg9XCER7aYy9a/j7Pg9P7CPs= +modernc.org/strutil v1.1.0 h1:+1/yCzZxY2pZwwrsbH+4T7BQMoLQ9QiBshRC9eicYsc= +modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +modernc.org/tcl v1.5.2 h1:sYNjGr4zK6cDH74USl8wVJRrvDX6UOLpG0j4lFvR0W0= +modernc.org/tcl v1.5.2/go.mod h1:pmJYOLgpiys3oI4AeAafkcUfE+TKKilminxNyU/+Zlo= +modernc.org/token v1.0.0 h1:a0jaWiNMDhDUtqOj09wvjWWAqd3q7WpBulmL9H2egsk= +modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.0.1-0.20210308123920-1f282aa71362/go.mod h1:8/SRk5C/HgiQWCgXdfpb+1RvhORdkz5sw72d3jjtyqA= +modernc.org/z v1.0.1 h1:WyIDpEpAIx4Hel6q/Pcgj/VhaQV5XPJ2I6ryIYbjnpc= +modernc.org/z v1.0.1/go.mod h1:8/SRk5C/HgiQWCgXdfpb+1RvhORdkz5sw72d3jjtyqA= +modernc.org/zappy v1.0.0 h1:dPVaP+3ueIUv4guk8PuZ2wiUGcJ1WUVvIheeSSTD0yk= +modernc.org/zappy v1.0.0/go.mod h1:hHe+oGahLVII/aTTyWK/b53VDHMAGCBYYeZ9sn83HC4= +rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1 h1:k1MczvYDUvJBe93bYd7wrZLLUEcLZAuF824/I4e5Xr4= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15 h1:4uqm9Mv+w2MmBYD+F4qf/v6tDFUdPOk29C095RbU5mY= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.0.3 h1:4oyYo8NREp49LBBhKxEqCulFjg26rawYKrnCmg+Sr6c= +sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/cli/build_aws-s3.go b/internal/cli/build_aws-s3.go similarity index 74% rename from cli/build_aws-s3.go rename to internal/cli/build_aws-s3.go index 9a6d09aa7..6b2a2e9dd 100644 --- a/cli/build_aws-s3.go +++ b/internal/cli/build_aws-s3.go @@ -1,6 +1,7 @@ +//go:build aws_s3 // +build aws_s3 -package main +package cli import ( _ "github.com/golang-migrate/migrate/v4/source/aws_s3" diff --git a/internal/cli/build_bitbucket.go b/internal/cli/build_bitbucket.go new file mode 100644 index 000000000..f939445d6 --- /dev/null +++ b/internal/cli/build_bitbucket.go @@ -0,0 +1,8 @@ +//go:build bitbucket +// +build bitbucket + +package cli + +import ( + _ "github.com/golang-migrate/migrate/v4/source/bitbucket" +) diff --git a/cli/build_cassandra.go b/internal/cli/build_cassandra.go similarity index 74% rename from cli/build_cassandra.go rename to internal/cli/build_cassandra.go index ba0650eb3..a24c300bb 100644 --- a/cli/build_cassandra.go +++ b/internal/cli/build_cassandra.go @@ -1,6 +1,7 @@ +//go:build cassandra // +build cassandra -package main +package cli import ( _ "github.com/golang-migrate/migrate/v4/database/cassandra" diff --git a/cli/build_clickhouse.go b/internal/cli/build_clickhouse.go similarity index 56% rename from cli/build_clickhouse.go rename to internal/cli/build_clickhouse.go index 2fd242967..c10d65c5e 100644 --- a/cli/build_clickhouse.go +++ b/internal/cli/build_clickhouse.go @@ -1,8 +1,9 @@ +//go:build clickhouse // +build clickhouse -package main +package cli import ( + _ "github.com/ClickHouse/clickhouse-go" _ "github.com/golang-migrate/migrate/v4/database/clickhouse" - _ "github.com/kshvakov/clickhouse" ) diff --git a/cli/build_cockroachdb.go b/internal/cli/build_cockroachdb.go similarity index 73% rename from cli/build_cockroachdb.go rename to internal/cli/build_cockroachdb.go index bc61df378..d132a4b3c 100644 --- a/cli/build_cockroachdb.go +++ b/internal/cli/build_cockroachdb.go @@ -1,6 +1,7 @@ +//go:build cockroachdb // +build cockroachdb -package main +package cli import ( _ "github.com/golang-migrate/migrate/v4/database/cockroachdb" diff --git a/internal/cli/build_firebird.go b/internal/cli/build_firebird.go new file mode 100644 index 000000000..e760e2f23 --- /dev/null +++ b/internal/cli/build_firebird.go @@ -0,0 +1,8 @@ +//go:build firebird +// +build firebird + +package cli + +import ( + _ "github.com/golang-migrate/migrate/v4/database/firebird" +) diff --git a/cli/build_github.go b/internal/cli/build_github.go similarity index 74% rename from cli/build_github.go rename to internal/cli/build_github.go index fa65ed1bd..89e316e2f 100644 --- a/cli/build_github.go +++ b/internal/cli/build_github.go @@ -1,6 +1,7 @@ +//go:build github // +build github -package main +package cli import ( _ "github.com/golang-migrate/migrate/v4/source/github" diff --git a/internal/cli/build_github_ee.go b/internal/cli/build_github_ee.go new file mode 100644 index 000000000..230931375 --- /dev/null +++ b/internal/cli/build_github_ee.go @@ -0,0 +1,8 @@ +//go:build github +// +build github + +package cli + +import ( + _ "github.com/golang-migrate/migrate/v4/source/github_ee" +) diff --git a/internal/cli/build_gitlab.go b/internal/cli/build_gitlab.go new file mode 100644 index 000000000..9224f1fd3 --- /dev/null +++ b/internal/cli/build_gitlab.go @@ -0,0 +1,8 @@ +//go:build gitlab +// +build gitlab + +package cli + +import ( + _ "github.com/golang-migrate/migrate/v4/source/gitlab" +) diff --git a/cli/build_go-bindata.go b/internal/cli/build_go-bindata.go similarity index 73% rename from cli/build_go-bindata.go rename to internal/cli/build_go-bindata.go index 4fe257cae..65fe121e3 100644 --- a/cli/build_go-bindata.go +++ b/internal/cli/build_go-bindata.go @@ -1,6 +1,7 @@ +//go:build go_bindata // +build go_bindata -package main +package cli import ( _ "github.com/golang-migrate/migrate/v4/source/go_bindata" diff --git a/cli/build_godoc-vfs.go b/internal/cli/build_godoc-vfs.go similarity index 73% rename from cli/build_godoc-vfs.go rename to internal/cli/build_godoc-vfs.go index 700d30a33..a12be05cb 100644 --- a/cli/build_godoc-vfs.go +++ b/internal/cli/build_godoc-vfs.go @@ -1,6 +1,7 @@ +//go:build godoc_vfs // +build godoc_vfs -package main +package cli import ( _ "github.com/golang-migrate/migrate/v4/source/godoc_vfs" diff --git a/cli/build_google-cloud-storage.go b/internal/cli/build_google-cloud-storage.go similarity index 72% rename from cli/build_google-cloud-storage.go rename to internal/cli/build_google-cloud-storage.go index f81ff761a..1aa841843 100644 --- a/cli/build_google-cloud-storage.go +++ b/internal/cli/build_google-cloud-storage.go @@ -1,6 +1,7 @@ +//go:build google_cloud_storage // +build google_cloud_storage -package main +package cli import ( _ "github.com/golang-migrate/migrate/v4/source/google_cloud_storage" diff --git a/internal/cli/build_mongodb.go b/internal/cli/build_mongodb.go new file mode 100644 index 000000000..6b9f8f231 --- /dev/null +++ b/internal/cli/build_mongodb.go @@ -0,0 +1,8 @@ +//go:build mongodb +// +build mongodb + +package cli + +import ( + _ "github.com/golang-migrate/migrate/v4/database/mongodb" +) diff --git a/cli/build_mysql.go b/internal/cli/build_mysql.go similarity index 74% rename from cli/build_mysql.go rename to internal/cli/build_mysql.go index 2c697cf56..3dbd53ccb 100644 --- a/cli/build_mysql.go +++ b/internal/cli/build_mysql.go @@ -1,6 +1,7 @@ +//go:build mysql // +build mysql -package main +package cli import ( _ "github.com/golang-migrate/migrate/v4/database/mysql" diff --git a/internal/cli/build_neo4j.go b/internal/cli/build_neo4j.go new file mode 100644 index 000000000..cf2d8488d --- /dev/null +++ b/internal/cli/build_neo4j.go @@ -0,0 +1,8 @@ +//go:build neo4j +// +build neo4j + +package cli + +import ( + _ "github.com/golang-migrate/migrate/v4/database/neo4j" +) diff --git a/internal/cli/build_pgx.go b/internal/cli/build_pgx.go new file mode 100644 index 000000000..41862e178 --- /dev/null +++ b/internal/cli/build_pgx.go @@ -0,0 +1,8 @@ +//go:build pgx +// +build pgx + +package cli + +import ( + _ "github.com/golang-migrate/migrate/v4/database/pgx" +) diff --git a/cli/build_postgres.go b/internal/cli/build_postgres.go similarity index 74% rename from cli/build_postgres.go rename to internal/cli/build_postgres.go index aa950d5d2..626cf5292 100644 --- a/cli/build_postgres.go +++ b/internal/cli/build_postgres.go @@ -1,6 +1,7 @@ +//go:build postgres // +build postgres -package main +package cli import ( _ "github.com/golang-migrate/migrate/v4/database/postgres" diff --git a/cli/build_ql.go b/internal/cli/build_ql.go similarity index 75% rename from cli/build_ql.go rename to internal/cli/build_ql.go index 21cd68f27..125c2cd3f 100644 --- a/cli/build_ql.go +++ b/internal/cli/build_ql.go @@ -1,6 +1,7 @@ +//go:build ql // +build ql -package main +package cli import ( _ "github.com/golang-migrate/migrate/v4/database/ql" diff --git a/cli/build_redshift.go b/internal/cli/build_redshift.go similarity index 74% rename from cli/build_redshift.go rename to internal/cli/build_redshift.go index 49fd7a2d4..a1617d1dc 100644 --- a/cli/build_redshift.go +++ b/internal/cli/build_redshift.go @@ -1,6 +1,7 @@ +//go:build redshift // +build redshift -package main +package cli import ( _ "github.com/golang-migrate/migrate/v4/database/redshift" diff --git a/internal/cli/build_snowflake.go b/internal/cli/build_snowflake.go new file mode 100644 index 000000000..8cd891947 --- /dev/null +++ b/internal/cli/build_snowflake.go @@ -0,0 +1,8 @@ +//go:build snowflake +// +build snowflake + +package cli + +import ( + _ "github.com/golang-migrate/migrate/v4/database/snowflake" +) diff --git a/cli/build_spanner.go b/internal/cli/build_spanner.go similarity index 74% rename from cli/build_spanner.go rename to internal/cli/build_spanner.go index b83139338..70b56fcca 100644 --- a/cli/build_spanner.go +++ b/internal/cli/build_spanner.go @@ -1,6 +1,7 @@ +//go:build spanner // +build spanner -package main +package cli import ( _ "github.com/golang-migrate/migrate/v4/database/spanner" diff --git a/internal/cli/build_sqlcipher.go b/internal/cli/build_sqlcipher.go new file mode 100644 index 000000000..1f6b31fd3 --- /dev/null +++ b/internal/cli/build_sqlcipher.go @@ -0,0 +1,8 @@ +//go:build sqlcipher +// +build sqlcipher + +package cli + +import ( + _ "github.com/golang-migrate/migrate/v4/database/sqlcipher" +) diff --git a/internal/cli/build_sqlite.go b/internal/cli/build_sqlite.go new file mode 100644 index 000000000..8c95657ce --- /dev/null +++ b/internal/cli/build_sqlite.go @@ -0,0 +1,8 @@ +//go:build sqlite +// +build sqlite + +package cli + +import ( + _ "github.com/golang-migrate/migrate/v4/database/sqlite" +) diff --git a/cli/build_sqlite3.go b/internal/cli/build_sqlite3.go similarity index 74% rename from cli/build_sqlite3.go rename to internal/cli/build_sqlite3.go index d74d66830..4d3867a61 100644 --- a/cli/build_sqlite3.go +++ b/internal/cli/build_sqlite3.go @@ -1,6 +1,7 @@ +//go:build sqlite3 // +build sqlite3 -package main +package cli import ( _ "github.com/golang-migrate/migrate/v4/database/sqlite3" diff --git a/internal/cli/build_sqlserver.go b/internal/cli/build_sqlserver.go new file mode 100644 index 000000000..f53b93bb0 --- /dev/null +++ b/internal/cli/build_sqlserver.go @@ -0,0 +1,8 @@ +//go:build sqlserver +// +build sqlserver + +package cli + +import ( + _ "github.com/golang-migrate/migrate/v4/database/sqlserver" +) diff --git a/internal/cli/commands.go b/internal/cli/commands.go new file mode 100644 index 000000000..7adec2f84 --- /dev/null +++ b/internal/cli/commands.go @@ -0,0 +1,248 @@ +package cli + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/golang-migrate/migrate/v4" + _ "github.com/golang-migrate/migrate/v4/database/stub" // TODO remove again + _ "github.com/golang-migrate/migrate/v4/source/file" +) + +var ( + errInvalidSequenceWidth = errors.New("Digits must be positive") + errIncompatibleSeqAndFormat = errors.New("The seq and format options are mutually exclusive") + errInvalidTimeFormat = errors.New("Time format may not be empty") +) + +func nextSeqVersion(matches []string, seqDigits int) (string, error) { + if seqDigits <= 0 { + return "", errInvalidSequenceWidth + } + + nextSeq := uint64(1) + + if len(matches) > 0 { + filename := matches[len(matches)-1] + matchSeqStr := filepath.Base(filename) + idx := strings.Index(matchSeqStr, "_") + + if idx < 1 { // Using 1 instead of 0 since there should be at least 1 digit + return "", fmt.Errorf("Malformed migration filename: %s", filename) + } + + var err error + matchSeqStr = matchSeqStr[0:idx] + nextSeq, err = strconv.ParseUint(matchSeqStr, 10, 64) + + if err != nil { + return "", err + } + + nextSeq++ + } + + version := fmt.Sprintf("%0[2]*[1]d", nextSeq, seqDigits) + + if len(version) > seqDigits { + return "", fmt.Errorf("Next sequence number %s too large. At most %d digits are allowed", version, seqDigits) + } + + return version, nil +} + +func timeVersion(startTime time.Time, format string) (version string, err error) { + switch format { + case "": + err = errInvalidTimeFormat + case "unix": + version = strconv.FormatInt(startTime.Unix(), 10) + case "unixNano": + version = strconv.FormatInt(startTime.UnixNano(), 10) + default: + version = startTime.Format(format) + } + + return +} + +// createCmd (meant to be called via a CLI command) creates a new migration +func createCmd(dir string, startTime time.Time, format string, name string, ext string, seq bool, seqDigits int, print bool) error { + if seq && format != defaultTimeFormat { + return errIncompatibleSeqAndFormat + } + + var version string + var err error + + dir = filepath.Clean(dir) + ext = "." + strings.TrimPrefix(ext, ".") + + if seq { + matches, err := filepath.Glob(filepath.Join(dir, "*"+ext)) + + if err != nil { + return err + } + + version, err = nextSeqVersion(matches, seqDigits) + + if err != nil { + return err + } + } else { + version, err = timeVersion(startTime, format) + + if err != nil { + return err + } + } + + versionGlob := filepath.Join(dir, version+"_*"+ext) + matches, err := filepath.Glob(versionGlob) + + if err != nil { + return err + } + + if len(matches) > 0 { + return fmt.Errorf("duplicate migration version: %s", version) + } + + if err = os.MkdirAll(dir, os.ModePerm); err != nil { + return err + } + + for _, direction := range []string{"up", "down"} { + basename := fmt.Sprintf("%s_%s.%s%s", version, name, direction, ext) + filename := filepath.Join(dir, basename) + + if err = createFile(filename); err != nil { + return err + } + + if print { + absPath, _ := filepath.Abs(filename) + log.Println(absPath) + } + } + + return nil +} + +func createFile(filename string) error { + // create exclusive (fails if file already exists) + // os.Create() specifies 0666 as the FileMode, so we're doing the same + f, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) + + if err != nil { + return err + } + + return f.Close() +} + +func gotoCmd(m *migrate.Migrate, v uint) error { + if err := m.Migrate(v); err != nil { + if err != migrate.ErrNoChange { + return err + } + log.Println(err) + } + return nil +} + +func upCmd(m *migrate.Migrate, limit int) error { + if limit >= 0 { + if err := m.Steps(limit); err != nil { + if err != migrate.ErrNoChange { + return err + } + log.Println(err) + } + } else { + if err := m.Up(); err != nil { + if err != migrate.ErrNoChange { + return err + } + log.Println(err) + } + } + return nil +} + +func downCmd(m *migrate.Migrate, limit int) error { + if limit >= 0 { + if err := m.Steps(-limit); err != nil { + if err != migrate.ErrNoChange { + return err + } + log.Println(err) + } + } else { + if err := m.Down(); err != nil { + if err != migrate.ErrNoChange { + return err + } + log.Println(err) + } + } + return nil +} + +func dropCmd(m *migrate.Migrate) error { + if err := m.Drop(); err != nil { + return err + } + return nil +} + +func forceCmd(m *migrate.Migrate, v int) error { + if err := m.Force(v); err != nil { + return err + } + return nil +} + +func versionCmd(m *migrate.Migrate) error { + v, dirty, err := m.Version() + if err != nil { + return err + } + if dirty { + log.Printf("%v (dirty)\n", v) + } else { + log.Println(v) + } + return nil +} + +// numDownMigrationsFromArgs returns an int for number of migrations to apply +// and a bool indicating if we need a confirm before applying +func numDownMigrationsFromArgs(applyAll bool, args []string) (int, bool, error) { + if applyAll { + if len(args) > 0 { + return 0, false, errors.New("-all cannot be used with other arguments") + } + return -1, false, nil + } + + switch len(args) { + case 0: + return -1, true, nil + case 1: + downValue := args[0] + n, err := strconv.ParseUint(downValue, 10, 64) + if err != nil { + return 0, false, errors.New("can't read limit argument N") + } + return int(n), false, nil + default: + return 0, false, errors.New("too many arguments") + } +} diff --git a/internal/cli/commands_test.go b/internal/cli/commands_test.go new file mode 100644 index 000000000..1705e479b --- /dev/null +++ b/internal/cli/commands_test.go @@ -0,0 +1,293 @@ +package cli + +import ( + "errors" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/suite" +) + +type CreateCmdSuite struct { + suite.Suite +} + +func TestCreateCmdSuite(t *testing.T) { + suite.Run(t, &CreateCmdSuite{}) +} + +func (s *CreateCmdSuite) mustCreateTempDir() string { + tmpDir, err := ioutil.TempDir("", "migrate_") + + if err != nil { + s.FailNow(err.Error()) + } + + return tmpDir +} + +func (s *CreateCmdSuite) mustCreateDir(dir string) { + if err := os.MkdirAll(dir, 0755); err != nil { + s.FailNow(err.Error()) + } +} + +func (s *CreateCmdSuite) mustRemoveDir(dir string) { + if err := os.RemoveAll(dir); err != nil { + s.FailNow(err.Error()) + } +} + +func (s *CreateCmdSuite) mustWriteFile(dir, file, body string) { + if err := ioutil.WriteFile(filepath.Join(dir, file), []byte(body), 0644); err != nil { + s.FailNow(err.Error()) + } +} + +func (s *CreateCmdSuite) mustGetwd() string { + cwd, err := os.Getwd() + + if err != nil { + s.FailNow(err.Error()) + } + + return cwd +} + +func (s *CreateCmdSuite) mustChdir(dir string) { + if err := os.Chdir(dir); err != nil { + s.FailNow(err.Error()) + } +} + +func (s *CreateCmdSuite) assertEmptyDir(dir string) bool { + fis, err := ioutil.ReadDir(dir) + + if err != nil { + return s.Fail(err.Error()) + } + + return s.Empty(fis) +} + +func (s *CreateCmdSuite) TestNextSeqVersion() { + cases := []struct { + tid string + matches []string + seqDigits int + expected string + expectedErr error + }{ + {"Bad digits", []string{}, 0, "", errInvalidSequenceWidth}, + {"Single digit initialize", []string{}, 1, "1", nil}, + {"Single digit malformed", []string{"bad"}, 1, "", errors.New("Malformed migration filename: bad")}, + {"Single digit no int", []string{"bad_bad"}, 1, "", errors.New(`strconv.ParseUint: parsing "bad": invalid syntax`)}, + {"Single digit negative seq", []string{"-5_test"}, 1, "", errors.New(`strconv.ParseUint: parsing "-5": invalid syntax`)}, + {"Single digit increment", []string{"3_test", "4_test"}, 1, "5", nil}, + {"Single digit overflow", []string{"9_test"}, 1, "", errors.New("Next sequence number 10 too large. At most 1 digits are allowed")}, + {"Zero-pad initialize", []string{}, 6, "000001", nil}, + {"Zero-pad malformed", []string{"bad"}, 6, "", errors.New("Malformed migration filename: bad")}, + {"Zero-pad no int", []string{"bad_bad"}, 6, "", errors.New(`strconv.ParseUint: parsing "bad": invalid syntax`)}, + {"Zero-pad negative seq", []string{"-000005_test"}, 6, "", errors.New(`strconv.ParseUint: parsing "-000005": invalid syntax`)}, + {"Zero-pad increment", []string{"000003_test", "000004_test"}, 6, "000005", nil}, + {"Zero-pad overflow", []string{"999999_test"}, 6, "", errors.New("Next sequence number 1000000 too large. At most 6 digits are allowed")}, + {"dir absolute path", []string{"/migrationDir/000001_test"}, 6, "000002", nil}, + {"dir relative path", []string{"migrationDir/000001_test"}, 6, "000002", nil}, + {"dir dot prefix", []string{"./migrationDir/000001_test"}, 6, "000002", nil}, + {"dir parent prefix", []string{"../migrationDir/000001_test"}, 6, "000002", nil}, + {"dir no prefix", []string{"000001_test"}, 6, "000002", nil}, + } + + for _, c := range cases { + s.Run(c.tid, func() { + v, err := nextSeqVersion(c.matches, c.seqDigits) + + if c.expectedErr != nil { + s.EqualError(err, c.expectedErr.Error()) + } else { + s.NoError(err) + s.Equal(c.expected, v) + } + }) + } +} + +func (s *CreateCmdSuite) TestTimeVersion() { + ts := time.Date(2000, 12, 25, 00, 01, 02, 3456789, time.UTC) + tsUnixStr := strconv.FormatInt(ts.Unix(), 10) + tsUnixNanoStr := strconv.FormatInt(ts.UnixNano(), 10) + + cases := []struct { + tid string + time time.Time + format string + expected string + expectedErr error + }{ + {"Bad format", ts, "", "", errInvalidTimeFormat}, + {"unix", ts, "unix", tsUnixStr, nil}, + {"unixNano", ts, "unixNano", tsUnixNanoStr, nil}, + {"custom ymthms", ts, "20060102150405", "20001225000102", nil}, + } + + for _, c := range cases { + s.Run(c.tid, func() { + v, err := timeVersion(c.time, c.format) + + if c.expectedErr != nil { + s.EqualError(err, c.expectedErr.Error()) + } else { + s.NoError(err) + s.Equal(c.expected, v) + } + }) + } +} + +// TestCreateCmd tests function createCmd. +// +// For each test case, it creates a temp dir as "sandbox" (called `baseDir`) and +// all path manipulations are relative to `baseDir`. +func (s *CreateCmdSuite) TestCreateCmd() { + ts := time.Date(2000, 12, 25, 00, 01, 02, 3456789, time.UTC) + tsUnixStr := strconv.FormatInt(ts.Unix(), 10) + tsUnixNanoStr := strconv.FormatInt(ts.UnixNano(), 10) + testCwd := s.mustGetwd() + + cases := []struct { + tid string + existingDirs []string // directory paths to create before test. relative to baseDir. + cwd string // path to chdir to before test. relative to baseDir. + existingFiles []string // file paths created before test. relative to baseDir. + expectedFiles []string // file paths expected to exist after test. paths relative to baseDir. + expectedErr error + dir string // `dir` parameter. if absolute path, will be converted to baseDir/dir. + startTime time.Time + format string + seq bool + seqDigits int + ext string + name string + }{ + {"seq and format", nil, "", nil, nil, errIncompatibleSeqAndFormat, ".", ts, "unix", true, 4, "sql", "name"}, + {"seq init dir dot", nil, "", nil, []string{"0001_name.up.sql", "0001_name.down.sql"}, nil, ".", ts, defaultTimeFormat, true, 4, "sql", "name"}, + {"seq init dir dot trailing slash", nil, "", nil, []string{"0001_name.up.sql", "0001_name.down.sql"}, nil, "./", ts, defaultTimeFormat, true, 4, "sql", "name"}, + {"seq init dir double dot", []string{"subdir"}, "subdir", nil, []string{"0001_name.up.sql", "0001_name.down.sql"}, nil, "..", ts, defaultTimeFormat, true, 4, "sql", "name"}, + {"seq init dir double dot trailing slash", []string{"subdir"}, "subdir", nil, []string{"0001_name.up.sql", "0001_name.down.sql"}, nil, "../", ts, defaultTimeFormat, true, 4, "sql", "name"}, + {"seq init dir absolute", []string{"subdir"}, "", nil, []string{"subdir/0001_name.up.sql", "subdir/0001_name.down.sql"}, nil, "/subdir", ts, defaultTimeFormat, true, 4, "sql", "name"}, + {"seq init dir absolute trailing slash", []string{"subdir"}, "", nil, []string{"subdir/0001_name.up.sql", "subdir/0001_name.down.sql"}, nil, "/subdir/", ts, defaultTimeFormat, true, 4, "sql", "name"}, + {"seq init dir relative", []string{"subdir"}, "", nil, []string{"subdir/0001_name.up.sql", "subdir/0001_name.down.sql"}, nil, "subdir", ts, defaultTimeFormat, true, 4, "sql", "name"}, + {"seq init dir relative trailing slash", []string{"subdir"}, "", nil, []string{"subdir/0001_name.up.sql", "subdir/0001_name.down.sql"}, nil, "subdir/", ts, defaultTimeFormat, true, 4, "sql", "name"}, + {"seq init dir dot relative", []string{"subdir"}, "", nil, []string{"subdir/0001_name.up.sql", "subdir/0001_name.down.sql"}, nil, "./subdir", ts, defaultTimeFormat, true, 4, "sql", "name"}, + {"seq init dir dot relative trailing slash", []string{"subdir"}, "", nil, []string{"subdir/0001_name.up.sql", "subdir/0001_name.down.sql"}, nil, "./subdir/", ts, defaultTimeFormat, true, 4, "sql", "name"}, + {"seq init dir double dot relative", []string{"subdir"}, "subdir", nil, []string{"subdir/0001_name.up.sql", "subdir/0001_name.down.sql"}, nil, "../subdir", ts, defaultTimeFormat, true, 4, "sql", "name"}, + {"seq init dir double dot relative trailing slash", []string{"subdir"}, "subdir", nil, []string{"subdir/0001_name.up.sql", "subdir/0001_name.down.sql"}, nil, "../subdir/", ts, defaultTimeFormat, true, 4, "sql", "name"}, + {"seq init dir maze", []string{"subdir"}, "subdir", nil, []string{"0001_name.up.sql", "0001_name.down.sql"}, nil, "..//subdir/./.././/subdir/..", ts, defaultTimeFormat, true, 4, "sql", "name"}, + {"seq width invalid", nil, "", nil, nil, errInvalidSequenceWidth, ".", ts, defaultTimeFormat, true, 0, "sql", "name"}, + {"seq malformed", nil, "", []string{"bad.sql"}, []string{"bad.sql"}, errors.New("Malformed migration filename: bad.sql"), ".", ts, defaultTimeFormat, true, 4, "sql", "name"}, + {"seq not int", nil, "", []string{"bad_bad.sql"}, []string{"bad_bad.sql"}, errors.New(`strconv.ParseUint: parsing "bad": invalid syntax`), ".", ts, defaultTimeFormat, true, 4, "sql", "name"}, + {"seq negative", nil, "", []string{"-5_negative.sql"}, []string{"-5_negative.sql"}, errors.New(`strconv.ParseUint: parsing "-5": invalid syntax`), ".", ts, defaultTimeFormat, true, 4, "sql", "name"}, + {"seq increment", nil, "", []string{"3_three.sql", "4_four.sql"}, []string{"3_three.sql", "4_four.sql", "0005_five.up.sql", "0005_five.down.sql"}, nil, ".", ts, defaultTimeFormat, true, 4, "sql", "five"}, + {"seq overflow", nil, "", []string{"9_nine.sql"}, []string{"9_nine.sql"}, errors.New(`Next sequence number 10 too large. At most 1 digits are allowed`), ".", ts, defaultTimeFormat, true, 1, "sql", "ten"}, + {"time empty format", nil, "", nil, nil, errInvalidTimeFormat, ".", ts, "", false, 0, "sql", "name"}, + {"time unix", nil, "", nil, []string{tsUnixStr + "_name.up.sql", tsUnixStr + "_name.down.sql"}, nil, ".", ts, "unix", false, 0, "sql", "name"}, + {"time unixNano", nil, "", nil, []string{tsUnixNanoStr + "_name.up.sql", tsUnixNanoStr + "_name.down.sql"}, nil, ".", ts, "unixNano", false, 0, "sql", "name"}, + {"time custom format", nil, "", nil, []string{"20001225000102_name.up.sql", "20001225000102_name.down.sql"}, nil, ".", ts, "20060102150405", false, 0, "sql", "name"}, + {"time version collision", nil, "", []string{"20001225_name.up.sql", "20001225_name.down.sql"}, []string{"20001225_name.up.sql", "20001225_name.down.sql"}, errors.New("duplicate migration version: 20001225"), ".", ts, "20060102", false, 0, "sql", "name"}, + {"dir invalid", nil, "", []string{"file"}, []string{"file"}, errors.New("mkdir 'test: this is invalid dir name'\x00: invalid argument"), "'test: this is invalid dir name'\000", ts, "unix", false, 0, "sql", "name"}, + } + + for _, c := range cases { + s.Run(c.tid, func() { + baseDir := s.mustCreateTempDir() + + for _, d := range c.existingDirs { + s.mustCreateDir(filepath.Join(baseDir, d)) + } + + cwd := baseDir + + if c.cwd != "" { + cwd = filepath.Join(baseDir, c.cwd) + } + + s.mustChdir(cwd) + + for _, f := range c.existingFiles { + s.mustWriteFile(baseDir, f, "") + } + + dir := c.dir + dir = filepath.ToSlash(dir) + volName := filepath.VolumeName(baseDir) + // Windows specific, can not recognize \subdir as abs path + isWindowsAbsPathNoLetter := strings.HasPrefix(dir, "/") && volName != "" + isRealAbsPath := filepath.IsAbs(dir) + if isWindowsAbsPathNoLetter || isRealAbsPath { + dir = filepath.Join(baseDir, dir) + } + + err := createCmd(dir, c.startTime, c.format, c.name, c.ext, c.seq, c.seqDigits, false) + + if c.expectedErr != nil { + s.EqualError(err, c.expectedErr.Error()) + } else { + s.NoError(err) + } + + if len(c.expectedFiles) == 0 { + s.assertEmptyDir(baseDir) + } else { + for _, f := range c.expectedFiles { + s.FileExists(filepath.Join(baseDir, f)) + } + } + + s.mustChdir(testCwd) + s.mustRemoveDir(baseDir) + }) + } +} + +func TestNumDownFromArgs(t *testing.T) { + cases := []struct { + name string + args []string + applyAll bool + expectedNeedConfirm bool + expectedNum int + expectedErrStr string + }{ + {"no args", []string{}, false, true, -1, ""}, + {"down all", []string{}, true, false, -1, ""}, + {"down 5", []string{"5"}, false, false, 5, ""}, + {"down N", []string{"N"}, false, false, 0, "can't read limit argument N"}, + {"extra arg after -all", []string{"5"}, true, false, 0, "-all cannot be used with other arguments"}, + {"extra arg before -all", []string{"5", "-all"}, false, false, 0, "too many arguments"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + num, needsConfirm, err := numDownMigrationsFromArgs(c.applyAll, c.args) + if needsConfirm != c.expectedNeedConfirm { + t.Errorf("Incorrect needsConfirm was: %v wanted %v", needsConfirm, c.expectedNeedConfirm) + } + + if num != c.expectedNum { + t.Errorf("Incorrect num was: %v wanted %v", num, c.expectedNum) + } + + if err != nil { + if err.Error() != c.expectedErrStr { + t.Error("Incorrect error: " + err.Error() + " != " + c.expectedErrStr) + } + } else if c.expectedErrStr != "" { + t.Error("Expected error: " + c.expectedErrStr + " but got nil instead") + } + }) + } +} diff --git a/cli/log.go b/internal/cli/log.go similarity index 77% rename from cli/log.go rename to internal/cli/log.go index e802fd2b5..91c6474f2 100644 --- a/cli/log.go +++ b/internal/cli/log.go @@ -1,4 +1,4 @@ -package main +package cli import ( "fmt" @@ -6,10 +6,12 @@ import ( "os" ) +// Log represents the logger type Log struct { verbose bool } +// Printf prints out formatted string into a log func (l *Log) Printf(format string, v ...interface{}) { if l.verbose { logpkg.Printf(format, v...) @@ -18,6 +20,7 @@ func (l *Log) Printf(format string, v ...interface{}) { } } +// Println prints out args into a log func (l *Log) Println(args ...interface{}) { if l.verbose { logpkg.Println(args...) @@ -26,15 +29,11 @@ func (l *Log) Println(args ...interface{}) { } } +// Verbose shows if verbose print enabled func (l *Log) Verbose() bool { return l.verbose } -func (l *Log) fatalf(format string, v ...interface{}) { - l.Printf(format, v...) - os.Exit(1) -} - func (l *Log) fatal(args ...interface{}) { l.Println(args...) os.Exit(1) diff --git a/internal/cli/main.go b/internal/cli/main.go new file mode 100644 index 000000000..af885e2de --- /dev/null +++ b/internal/cli/main.go @@ -0,0 +1,403 @@ +package cli + +import ( + "fmt" + "net/url" + "os" + "os/signal" + "strconv" + "strings" + "syscall" + "time" + + flag "github.com/spf13/pflag" + "github.com/spf13/viper" + + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database" + "github.com/golang-migrate/migrate/v4/source" +) + +const ( + defaultTimeFormat = "20060102150405" + defaultTimezone = "UTC" + createUsage = `create [-ext E] [-dir D] [-seq] [-digits N] [-format] [-tz] NAME + Create a set of timestamped up/down migrations titled NAME, in directory D with extension E. + Use -seq option to generate sequential up/down migrations with N digits. + Use -format option to specify a Go time format string. Note: migrations with the same time cause "duplicate migration version" error. + Use -tz option to specify the timezone that will be used when generating non-sequential migrations (defaults: UTC). +` + gotoUsage = `goto V Migrate to version V` + upUsage = `up [N] Apply all or N up migrations` + downUsage = `down [N] [-all] Apply all or N down migrations + Use -all to apply all down migrations` + dropUsage = `drop [-f] Drop everything inside database + Use -f to bypass confirmation` + forceUsage = `force V Set version V but don't run migration (ignores dirty state)` +) + +func handleSubCmdHelp(help bool, usage string, flagSet *flag.FlagSet) { + if help { + fmt.Fprintln(os.Stderr, usage) + flagSet.PrintDefaults() + os.Exit(0) + } +} + +func newFlagSetWithHelp(name string) (*flag.FlagSet, *bool) { + flagSet := flag.NewFlagSet(name, flag.ExitOnError) + helpPtr := flagSet.Bool("help", false, "Print help information") + return flagSet, helpPtr +} + +// set main log +var log = &Log{} + +func printUsageAndExit() { + flag.Usage() + + // If a command is not found we exit with a status 2 to match the behavior + // of flag.Parse() with flag.ExitOnError when parsing an invalid flag. + os.Exit(2) +} + +func dbMakeConnectionString(driver, user, password, address, name, ssl string) string { + return fmt.Sprintf("%s://%s:%s@%s/%s?sslmode=%s", + driver, url.QueryEscape(user), url.QueryEscape(password), address, name, ssl, + ) +} + +// Main function of a cli application. It is public for backwards compatibility with `cli` package +func Main(version string) { + help := viper.GetBool("help") + version = viper.GetString("version") + verbose := viper.GetBool("verbose") + prefetch := viper.GetInt("prefetch") + lockTimeout := viper.GetInt("lock-timeout") + path := viper.GetString("path") + sourcePtr := viper.GetString("source") + + databasePtr := viper.GetString("database.dsn") + if databasePtr == "" { + databasePtr = dbMakeConnectionString( + viper.GetString("database.driver"), viper.GetString("database.user"), + viper.GetString("database.password"), viper.GetString("database.address"), + viper.GetString("database.name"), viper.GetString("database.ssl"), + ) + } + + flag.Usage = func() { + fmt.Fprintf(os.Stderr, + `Usage: migrate OPTIONS COMMAND [arg...] + migrate [ -version | -help ] + +Options: + --source Location of the migrations (driver://url) + --path Shorthand for -source=file://path + --database Run migrations against this database (driver://url) + --prefetch N Number of migrations to load in advance before executing (default 10) + --lock-timeout N Allow N seconds to acquire database lock (default 15) + --verbose Print verbose logging + --version Print version + --help Print usage + + // Infoblox specific + --config.source directory of the configuration file (default "/cli/config") + --config.file configuration file name (without extension) + --database.dsn database connection string + --database.driver database driver (default postgres) + --database.address address of the database (default "0.0.0.0:5432") + --database.name name of the database + --database.user database username (default "postgres") + --database.password database password (default "postgres") + --database.ssl database ssl mode (default "disable") + +Commands: + %s + %s + %s + %s + %s + %s + version Print current migration version + +Source drivers: `+strings.Join(source.List(), ", ")+` +Database drivers: `+strings.Join(database.List(), ", ")+"\n", createUsage, gotoUsage, upUsage, downUsage, dropUsage, forceUsage) + } + + // initialize logger + log.verbose = verbose + + // show cli version + if version == "" { + fmt.Fprintln(os.Stderr, version) + os.Exit(0) + } + + // show help + if help { + flag.Usage() + os.Exit(0) + } + + // translate -path into -source if given + if sourcePtr == "" && path != "" { + sourcePtr = fmt.Sprintf("file://%v", path) + } + + // initialize migrate + // don't catch migraterErr here and let each command decide + // how it wants to handle the error + migrater, migraterErr := migrate.New(sourcePtr, databasePtr) + defer func() { + if migraterErr == nil { + if _, err := migrater.Close(); err != nil { + log.Println(err) + } + } + }() + if migraterErr == nil { + migrater.Log = log + migrater.PrefetchMigrations = uint(prefetch) + migrater.LockTimeout = time.Duration(int64(lockTimeout)) * time.Second + + // handle Ctrl+c + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT) + go func() { + for range signals { + log.Println("Stopping after this running migration ...") + migrater.GracefulStop <- true + return + } + }() + } + + startTime := time.Now() + + if len(flag.Args()) < 1 { + printUsageAndExit() + } + args := flag.Args()[1:] + + switch flag.Arg(0) { + case "create": + + seq := false + seqDigits := 6 + + createFlagSet, help := newFlagSetWithHelp("create") + extPtr := createFlagSet.String("ext", "", "File extension") + dirPtr := createFlagSet.String("dir", "", "Directory to place file in (default: current working directory)") + formatPtr := createFlagSet.String("format", defaultTimeFormat, `The Go time format string to use. If the string "unix" or "unixNano" is specified, then the seconds or nanoseconds since January 1, 1970 UTC respectively will be used. Caution, due to the behavior of time.Time.Format(), invalid format strings will not error`) + timezoneName := createFlagSet.String("tz", defaultTimezone, `The timezone that will be used for generating timestamps (default: utc)`) + createFlagSet.BoolVar(&seq, "seq", seq, "Use sequential numbers instead of timestamps (default: false)") + createFlagSet.IntVar(&seqDigits, "digits", seqDigits, "The number of digits to use in sequences (default: 6)") + + if err := createFlagSet.Parse(args); err != nil { + log.fatalErr(err) + } + + handleSubCmdHelp(*help, createUsage, createFlagSet) + + if createFlagSet.NArg() == 0 { + log.fatal("error: please specify name") + } + name := createFlagSet.Arg(0) + + if *extPtr == "" { + log.fatal("error: -ext flag must be specified") + } + + timezone, err := time.LoadLocation(*timezoneName) + if err != nil { + log.fatal(err) + } + + if err := createCmd(*dirPtr, startTime.In(timezone), *formatPtr, name, *extPtr, seq, seqDigits, true); err != nil { + log.fatalErr(err) + } + + case "goto": + + gotoSet, helpPtr := newFlagSetWithHelp("goto") + + if err := gotoSet.Parse(args); err != nil { + log.fatalErr(err) + } + + handleSubCmdHelp(*helpPtr, gotoUsage, gotoSet) + + if migraterErr != nil { + log.fatalErr(migraterErr) + } + + if gotoSet.NArg() == 0 { + log.fatal("error: please specify version argument V") + } + + v, err := strconv.ParseUint(gotoSet.Arg(0), 10, 64) + if err != nil { + log.fatal("error: can't read version argument V") + } + + if err := gotoCmd(migrater, uint(v)); err != nil { + log.fatalErr(err) + } + + if log.verbose { + log.Println("Finished after", time.Since(startTime)) + } + + case "up": + upSet, helpPtr := newFlagSetWithHelp("up") + + if err := upSet.Parse(args); err != nil { + log.fatalErr(err) + } + + handleSubCmdHelp(*helpPtr, upUsage, upSet) + + if migraterErr != nil { + log.fatalErr(migraterErr) + } + + limit := -1 + if upSet.NArg() > 0 { + n, err := strconv.ParseUint(upSet.Arg(0), 10, 64) + if err != nil { + log.fatal("error: can't read limit argument N") + } + limit = int(n) + } + + if err := upCmd(migrater, limit); err != nil { + log.fatalErr(err) + } + + if log.verbose { + log.Println("Finished after", time.Since(startTime)) + } + + case "down": + downFlagSet, helpPtr := newFlagSetWithHelp("down") + applyAll := downFlagSet.Bool("all", false, "Apply all down migrations") + + if err := downFlagSet.Parse(args); err != nil { + log.fatalErr(err) + } + + handleSubCmdHelp(*helpPtr, downUsage, downFlagSet) + + if migraterErr != nil { + log.fatalErr(migraterErr) + } + + downArgs := downFlagSet.Args() + num, needsConfirm, err := numDownMigrationsFromArgs(*applyAll, downArgs) + if err != nil { + log.fatalErr(err) + } + if needsConfirm { + log.Println("Are you sure you want to apply all down migrations? [y/N]") + var response string + fmt.Scanln(&response) + response = strings.ToLower(strings.TrimSpace(response)) + + if response == "y" { + log.Println("Applying all down migrations") + } else { + log.fatal("Not applying all down migrations") + } + } + + if err := downCmd(migrater, num); err != nil { + log.fatalErr(err) + } + + if log.verbose { + log.Println("Finished after", time.Since(startTime)) + } + + case "drop": + dropFlagSet, help := newFlagSetWithHelp("drop") + forceDrop := dropFlagSet.Bool("f", false, "Force the drop command by bypassing the confirmation prompt") + + if err := dropFlagSet.Parse(args); err != nil { + log.fatalErr(err) + } + + handleSubCmdHelp(*help, dropUsage, dropFlagSet) + + if !*forceDrop { + log.Println("Are you sure you want to drop the entire database schema? [y/N]") + var response string + fmt.Scanln(&response) + response = strings.ToLower(strings.TrimSpace(response)) + + if response == "y" { + log.Println("Dropping the entire database schema") + } else { + log.fatal("Aborted dropping the entire database schema") + } + } + + if migraterErr != nil { + log.fatalErr(migraterErr) + } + + if err := dropCmd(migrater); err != nil { + log.fatalErr(err) + } + + if log.verbose { + log.Println("Finished after", time.Since(startTime)) + } + + case "force": + forceSet, helpPtr := newFlagSetWithHelp("force") + + if err := forceSet.Parse(args); err != nil { + log.fatalErr(err) + } + + handleSubCmdHelp(*helpPtr, forceUsage, forceSet) + + if migraterErr != nil { + log.fatalErr(migraterErr) + } + + if forceSet.NArg() == 0 { + log.fatal("error: please specify version argument V") + } + + v, err := strconv.ParseInt(forceSet.Arg(0), 10, 64) + if err != nil { + log.fatal("error: can't read version argument V") + } + + if v < -1 { + log.fatal("error: argument V must be >= -1") + } + + if err := forceCmd(migrater, int(v)); err != nil { + log.fatalErr(err) + } + + if log.verbose { + log.Println("Finished after", time.Since(startTime)) + } + + case "version": + if migraterErr != nil { + log.fatalErr(migraterErr) + } + + if err := versionCmd(migrater); err != nil { + log.fatalErr(err) + } + + default: + printUsageAndExit() + } +} diff --git a/internal/url/url.go b/internal/url/url.go new file mode 100644 index 000000000..e793fa828 --- /dev/null +++ b/internal/url/url.go @@ -0,0 +1,25 @@ +package url + +import ( + "errors" + "strings" +) + +var errNoScheme = errors.New("no scheme") +var errEmptyURL = errors.New("URL cannot be empty") + +// schemeFromURL returns the scheme from a URL string +func SchemeFromURL(url string) (string, error) { + if url == "" { + return "", errEmptyURL + } + + i := strings.Index(url, ":") + + // No : or : is the first character. + if i < 1 { + return "", errNoScheme + } + + return url[0:i], nil +} diff --git a/internal/url/url_test.go b/internal/url/url_test.go new file mode 100644 index 000000000..de338e76b --- /dev/null +++ b/internal/url/url_test.go @@ -0,0 +1,48 @@ +package url + +import ( + "testing" +) + +func TestSchemeFromUrl(t *testing.T) { + cases := []struct { + name string + urlStr string + expected string + expectErr error + }{ + { + name: "Simple", + urlStr: "protocol://path", + expected: "protocol", + }, + { + // See issue #264 + name: "MySQLWithPort", + urlStr: "mysql://user:pass@tcp(host:1337)/db", + expected: "mysql", + }, + { + name: "Empty", + urlStr: "", + expectErr: errEmptyURL, + }, + { + name: "NoScheme", + urlStr: "hello", + expectErr: errNoScheme, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s, err := SchemeFromURL(tc.urlStr) + if err != tc.expectErr { + t.Fatalf("expected %q, but received %q", tc.expectErr, err) + } + if s != tc.expected { + t.Fatalf("expected %q, but received %q", tc.expected, s) + } + }) + } +} diff --git a/migrate.go b/migrate.go index 8c8adfb5e..a57dceb5d 100644 --- a/migrate.go +++ b/migrate.go @@ -1,16 +1,20 @@ // Package migrate reads migrations from sources and runs them against databases. // Sources are defined by the `source.Driver` and databases by the `database.Driver` -// interface. The driver interfaces are kept "dump", all migration logic is kept +// interface. The driver interfaces are kept "dumb", all migration logic is kept // in this package. package migrate import ( + "errors" "fmt" "os" "sync" "time" + "github.com/hashicorp/go-multierror" + "github.com/golang-migrate/migrate/v4/database" + iurl "github.com/golang-migrate/migrate/v4/internal/url" "github.com/golang-migrate/migrate/v4/source" ) @@ -25,10 +29,11 @@ var DefaultPrefetchMigrations = uint(10) var DefaultLockTimeout = 15 * time.Second var ( - ErrNoChange = fmt.Errorf("no change") - ErrNilVersion = fmt.Errorf("no migration") - ErrLocked = fmt.Errorf("database locked") - ErrLockTimeout = fmt.Errorf("timeout: can't acquire database lock") + ErrNoChange = errors.New("no change") + ErrNilVersion = errors.New("no migration") + ErrInvalidVersion = errors.New("version must be >= -1") + ErrLocked = errors.New("database locked") + ErrLockTimeout = errors.New("timeout: can't acquire database lock") ) // ErrShortLimit is an error returned when not enough migrations @@ -62,11 +67,11 @@ type Migrate struct { // GracefulStop accepts `true` and will stop executing migrations // as soon as possible at a safe break point, so that the database // is not corrupted. - GracefulStop chan bool - isGracefulStop bool + GracefulStop chan bool + isLockedMu *sync.Mutex - isLockedMu *sync.Mutex - isLocked bool + isGracefulStop bool + isLocked bool // PrefetchMigrations defaults to DefaultPrefetchMigrations, // but can be set per Migrate instance. @@ -79,28 +84,28 @@ type Migrate struct { // New returns a new Migrate instance from a source URL and a database URL. // The URL scheme is defined by each driver. -func New(sourceUrl, databaseUrl string) (*Migrate, error) { +func New(sourceURL, databaseURL string) (*Migrate, error) { m := newCommon() - sourceName, err := sourceSchemeFromUrl(sourceUrl) + sourceName, err := iurl.SchemeFromURL(sourceURL) if err != nil { return nil, err } m.sourceName = sourceName - databaseName, err := databaseSchemeFromUrl(databaseUrl) + databaseName, err := iurl.SchemeFromURL(databaseURL) if err != nil { return nil, err } m.databaseName = databaseName - sourceDrv, err := source.Open(sourceUrl) + sourceDrv, err := source.Open(sourceURL) if err != nil { return nil, err } m.sourceDrv = sourceDrv - databaseDrv, err := database.Open(databaseUrl) + databaseDrv, err := database.Open(databaseURL) if err != nil { return nil, err } @@ -113,10 +118,10 @@ func New(sourceUrl, databaseUrl string) (*Migrate, error) { // and an existing database instance. The source URL scheme is defined by each driver. // Use any string that can serve as an identifier during logging as databaseName. // You are responsible for closing the underlying database client if necessary. -func NewWithDatabaseInstance(sourceUrl string, databaseName string, databaseInstance database.Driver) (*Migrate, error) { +func NewWithDatabaseInstance(sourceURL string, databaseName string, databaseInstance database.Driver) (*Migrate, error) { m := newCommon() - sourceName, err := schemeFromUrl(sourceUrl) + sourceName, err := iurl.SchemeFromURL(sourceURL) if err != nil { return nil, err } @@ -124,7 +129,7 @@ func NewWithDatabaseInstance(sourceUrl string, databaseName string, databaseInst m.databaseName = databaseName - sourceDrv, err := source.Open(sourceUrl) + sourceDrv, err := source.Open(sourceURL) if err != nil { return nil, err } @@ -139,10 +144,10 @@ func NewWithDatabaseInstance(sourceUrl string, databaseName string, databaseInst // and a database URL. The database URL scheme is defined by each driver. // Use any string that can serve as an identifier during logging as sourceName. // You are responsible for closing the underlying source client if necessary. -func NewWithSourceInstance(sourceName string, sourceInstance source.Driver, databaseUrl string) (*Migrate, error) { +func NewWithSourceInstance(sourceName string, sourceInstance source.Driver, databaseURL string) (*Migrate, error) { m := newCommon() - databaseName, err := schemeFromUrl(databaseUrl) + databaseName, err := iurl.SchemeFromURL(databaseURL) if err != nil { return nil, err } @@ -150,7 +155,7 @@ func NewWithSourceInstance(sourceName string, sourceInstance source.Driver, data m.sourceName = sourceName - databaseDrv, err := database.Open(databaseUrl) + databaseDrv, err := database.Open(databaseURL) if err != nil { return nil, err } @@ -345,7 +350,11 @@ func (m *Migrate) Run(migration ...*Migration) error { } ret <- migr - go migr.Buffer() + go func(migr *Migration) { + if err := migr.Buffer(); err != nil { + m.logErr(err) + } + }(migr) } }() @@ -357,7 +366,7 @@ func (m *Migrate) Run(migration ...*Migration) error { // It resets the dirty state to false. func (m *Migrate) Force(version int) error { if version < -1 { - panic("version must be >= -1") + return ErrInvalidVersion } if err := m.lock(); err != nil { @@ -432,7 +441,12 @@ func (m *Migrate) read(from int, to int, ret chan<- interface{}) { } ret <- migr - go migr.Buffer() + go func() { + if err := migr.Buffer(); err != nil { + m.logErr(err) + } + }() + from = int(firstVersion) } @@ -455,7 +469,12 @@ func (m *Migrate) read(from int, to int, ret chan<- interface{}) { } ret <- migr - go migr.Buffer() + go func() { + if err := migr.Buffer(); err != nil { + m.logErr(err) + } + }() + from = int(next) } @@ -468,7 +487,7 @@ func (m *Migrate) read(from int, to int, ret chan<- interface{}) { } prev, err := m.sourceDrv.Prev(suint(from)) - if os.IsNotExist(err) && to == -1 { + if errors.Is(err, os.ErrNotExist) && to == -1 { // apply nil migration migr, err := m.newMigration(suint(from), -1) if err != nil { @@ -476,7 +495,12 @@ func (m *Migrate) read(from int, to int, ret chan<- interface{}) { return } ret <- migr - go migr.Buffer() + go func() { + if err := migr.Buffer(); err != nil { + m.logErr(err) + } + }() + return } else if err != nil { @@ -491,7 +515,12 @@ func (m *Migrate) read(from int, to int, ret chan<- interface{}) { } ret <- migr - go migr.Buffer() + go func() { + if err := migr.Buffer(); err != nil { + m.logErr(err) + } + }() + from = int(prev) } } @@ -539,7 +568,11 @@ func (m *Migrate) readUp(from int, limit int, ret chan<- interface{}) { } ret <- migr - go migr.Buffer() + go func() { + if err := migr.Buffer(); err != nil { + m.logErr(err) + } + }() from = int(firstVersion) count++ continue @@ -547,7 +580,7 @@ func (m *Migrate) readUp(from int, limit int, ret chan<- interface{}) { // apply next migration next, err := m.sourceDrv.Next(suint(from)) - if os.IsNotExist(err) { + if errors.Is(err, os.ErrNotExist) { // no limit, but no migrations applied? if limit == -1 && count == 0 { ret <- ErrNoChange @@ -583,7 +616,11 @@ func (m *Migrate) readUp(from int, limit int, ret chan<- interface{}) { } ret <- migr - go migr.Buffer() + go func() { + if err := migr.Buffer(); err != nil { + m.logErr(err) + } + }() from = int(next) count++ } @@ -629,7 +666,7 @@ func (m *Migrate) readDown(from int, limit int, ret chan<- interface{}) { } prev, err := m.sourceDrv.Prev(suint(from)) - if os.IsNotExist(err) { + if errors.Is(err, os.ErrNotExist) { // no limit or haven't reached limit, apply "first" migration if limit == -1 || limit-count > 0 { firstVersion, err := m.sourceDrv.First() @@ -644,7 +681,11 @@ func (m *Migrate) readDown(from int, limit int, ret chan<- interface{}) { return } ret <- migr - go migr.Buffer() + go func() { + if err := migr.Buffer(); err != nil { + m.logErr(err) + } + }() count++ } @@ -665,7 +706,11 @@ func (m *Migrate) readDown(from int, limit int, ret chan<- interface{}) { } ret <- migr - go migr.Buffer() + go func() { + if err := migr.Buffer(); err != nil { + m.logErr(err) + } + }() from = int(prev) count++ } @@ -684,12 +729,12 @@ func (m *Migrate) runMigrations(ret <-chan interface{}) error { return nil } - switch r.(type) { + switch r := r.(type) { case error: - return r.(error) + return r case *Migration: - migr := r.(*Migration) + migr := r // set version with dirty state if err := m.databaseDrv.SetVersion(migr.TargetVersion, true); err != nil { @@ -722,7 +767,7 @@ func (m *Migrate) runMigrations(ret <-chan interface{}) error { } default: - panic("unknown type") + return fmt.Errorf("unknown type: %T with value: %+v", r, r) } } return nil @@ -730,30 +775,40 @@ func (m *Migrate) runMigrations(ret <-chan interface{}) error { // versionExists checks the source if either the up or down migration for // the specified migration version exists. -func (m *Migrate) versionExists(version uint) error { +func (m *Migrate) versionExists(version uint) (result error) { // try up migration first up, _, err := m.sourceDrv.ReadUp(version) if err == nil { - defer up.Close() + defer func() { + if errClose := up.Close(); errClose != nil { + result = multierror.Append(result, errClose) + } + }() } - if os.IsExist(err) { + if errors.Is(err, os.ErrExist) { return nil - } else if !os.IsNotExist(err) { + } else if !errors.Is(err, os.ErrNotExist) { return err } // then try down migration down, _, err := m.sourceDrv.ReadDown(version) if err == nil { - defer down.Close() + defer func() { + if errClose := down.Close(); errClose != nil { + result = multierror.Append(result, errClose) + } + }() } - if os.IsExist(err) { + if errors.Is(err, os.ErrExist) { return nil - } else if !os.IsNotExist(err) { + } else if !errors.Is(err, os.ErrNotExist) { return err } - return os.ErrNotExist + err = fmt.Errorf("no migration found for version %d: %w", version, err) + m.logErr(err) + return err } // stop returns true if no more migrations should be run against the database @@ -781,7 +836,7 @@ func (m *Migrate) newMigration(version uint, targetVersion int) (*Migration, err if targetVersion >= int(version) { r, identifier, err := m.sourceDrv.ReadUp(version) - if os.IsNotExist(err) { + if errors.Is(err, os.ErrNotExist) { // create "empty" migration migr, err = NewMigration(nil, "", version, targetVersion) if err != nil { @@ -801,7 +856,7 @@ func (m *Migrate) newMigration(version uint, targetVersion int) (*Migration, err } else { r, identifier, err := m.sourceDrv.ReadDown(version) - if os.IsNotExist(err) { + if errors.Is(err, os.ErrNotExist) { // create "empty" migration migr, err = NewMigration(nil, "", version, targetVersion) if err != nil { @@ -869,10 +924,9 @@ func (m *Migrate) lock() error { } else { errchan <- nil } - return }() - // wait until we either recieve ErrLockTimeout or error from Lock operation + // wait until we either receive ErrLockTimeout or error from Lock operation err := <-errchan if err == nil { m.isLocked = true @@ -900,7 +954,7 @@ func (m *Migrate) unlock() error { // if a prevErr is not nil. func (m *Migrate) unlockErr(prevErr error) error { if err := m.unlock(); err != nil { - return NewMultiError(prevErr, err) + return multierror.Append(prevErr, err) } return prevErr } @@ -918,3 +972,10 @@ func (m *Migrate) logVerbosePrintf(format string, v ...interface{}) { m.Log.Printf(format, v...) } } + +// logErr writes error to m.Log if not nil +func (m *Migrate) logErr(err error) { + if m.Log != nil { + m.Log.Printf("error: %v", err) + } +} diff --git a/migrate_test.go b/migrate_test.go index 7a474e929..5ed2d5e64 100644 --- a/migrate_test.go +++ b/migrate_test.go @@ -3,11 +3,15 @@ package migrate import ( "bytes" "database/sql" + "errors" "io/ioutil" "log" "os" + "strings" "testing" +) +import ( dStub "github.com/golang-migrate/migrate/v4/database/stub" "github.com/golang-migrate/migrate/v4/source" sStub "github.com/golang-migrate/migrate/v4/source/stub" @@ -19,6 +23,11 @@ import ( // | u d | - | u | u d | d | - | u d | var sourceStubMigrations *source.Migrations +const ( + srcDrvNameStub = "stub" + dbDrvNameStub = "stub" +) + func init() { sourceStubMigrations = source.NewMigrations() sourceStubMigrations.Append(&source.Migration{Version: 1, Direction: source.Up, Identifier: "CREATE 1"}) @@ -39,14 +48,14 @@ func TestNew(t *testing.T) { t.Fatal(err) } - if m.sourceName != "stub" { + if m.sourceName != srcDrvNameStub { t.Errorf("expected stub, got %v", m.sourceName) } if m.sourceDrv == nil { t.Error("expected sourceDrv not to be nil") } - if m.databaseName != "stub" { + if m.databaseName != dbDrvNameStub { t.Errorf("expected stub, got %v", m.databaseName) } if m.databaseDrv == nil { @@ -62,7 +71,7 @@ func ExampleNew() { } // Migrate all the way up ... - if err := m.Up(); err != nil { + if err := m.Up(); err != nil && err != ErrNoChange { log.Fatal(err) } } @@ -74,19 +83,19 @@ func TestNewWithDatabaseInstance(t *testing.T) { t.Fatal(err) } - m, err := NewWithDatabaseInstance("stub://", "stub", dbInst) + m, err := NewWithDatabaseInstance("stub://", dbDrvNameStub, dbInst) if err != nil { t.Fatal(err) } - if m.sourceName != "stub" { + if m.sourceName != srcDrvNameStub { t.Errorf("expected stub, got %v", m.sourceName) } if m.sourceDrv == nil { t.Error("expected sourceDrv not to be nil") } - if m.databaseName != "stub" { + if m.databaseName != dbDrvNameStub { t.Errorf("expected stub, got %v", m.databaseName) } if m.databaseDrv == nil { @@ -100,7 +109,11 @@ func ExampleNewWithDatabaseInstance() { if err != nil { log.Fatal(err) } - defer db.Close() + defer func() { + if err := db.Close(); err != nil { + log.Fatal(err) + } + }() // Create driver instance from db. // Check each driver if it supports the WithInstance function. @@ -129,19 +142,19 @@ func TestNewWithSourceInstance(t *testing.T) { t.Fatal(err) } - m, err := NewWithSourceInstance("stub", sInst, "stub://") + m, err := NewWithSourceInstance(srcDrvNameStub, sInst, "stub://") if err != nil { t.Fatal(err) } - if m.sourceName != "stub" { + if m.sourceName != srcDrvNameStub { t.Errorf("expected stub, got %v", m.sourceName) } if m.sourceDrv == nil { t.Error("expected sourceDrv not to be nil") } - if m.databaseName != "stub" { + if m.databaseName != dbDrvNameStub { t.Errorf("expected stub, got %v", m.databaseName) } if m.databaseDrv == nil { @@ -161,7 +174,7 @@ func ExampleNewWithSourceInstance() { } // Read migrations from Stub and connect to a local postgres database. - m, err := NewWithSourceInstance("stub", instance, "postgres://mattes:secret@localhost:5432/database?sslmode=disable") + m, err := NewWithSourceInstance(srcDrvNameStub, instance, "postgres://mattes:secret@localhost:5432/database?sslmode=disable") if err != nil { log.Fatal(err) } @@ -185,19 +198,19 @@ func TestNewWithInstance(t *testing.T) { t.Fatal(err) } - m, err := NewWithInstance("stub", sInst, "stub", dbInst) + m, err := NewWithInstance(srcDrvNameStub, sInst, dbDrvNameStub, dbInst) if err != nil { t.Fatal(err) } - if m.sourceName != "stub" { + if m.sourceName != srcDrvNameStub { t.Errorf("expected stub, got %v", m.sourceName) } if m.sourceDrv == nil { t.Error("expected sourceDrv not to be nil") } - if m.databaseName != "stub" { + if m.databaseName != dbDrvNameStub { t.Errorf("expected stub, got %v", m.databaseName) } if m.databaseDrv == nil { @@ -456,7 +469,7 @@ func TestMigrate(t *testing.T) { for i, v := range tt { err := m.Migrate(v.version) - if (v.expectErr == os.ErrNotExist && !os.IsNotExist(err)) || + if (v.expectErr == os.ErrNotExist && !errors.Is(err, os.ErrNotExist)) || (v.expectErr != os.ErrNotExist && err != v.expectErr) { t.Errorf("expected err %v, got %v, in %v", v.expectErr, err, i) @@ -719,7 +732,7 @@ func TestSteps(t *testing.T) { for i, v := range tt { err := m.Steps(v.steps) - if (v.expectErr == os.ErrNotExist && !os.IsNotExist(err)) || + if (v.expectErr == os.ErrNotExist && !errors.Is(err, os.ErrNotExist)) || (v.expectErr != os.ErrNotExist && err != v.expectErr) { t.Errorf("expected err %v, got %v, in %v", v.expectErr, err, i) @@ -1131,7 +1144,7 @@ func TestRead(t *testing.T) { go m.read(v.from, v.to, ret) migrations, err := migrationsFromChannel(ret) - if (v.expectErr == os.ErrNotExist && !os.IsNotExist(err)) || + if (v.expectErr == os.ErrNotExist && !errors.Is(err, os.ErrNotExist)) || (v.expectErr != os.ErrNotExist && v.expectErr != err) { t.Errorf("expected %v, got %v, in %v", v.expectErr, err, i) t.Logf("%v, in %v", migrations, i) @@ -1208,7 +1221,7 @@ func TestReadUp(t *testing.T) { go m.readUp(v.from, v.limit, ret) migrations, err := migrationsFromChannel(ret) - if (v.expectErr == os.ErrNotExist && !os.IsNotExist(err)) || + if (v.expectErr == os.ErrNotExist && !errors.Is(err, os.ErrNotExist)) || (v.expectErr != os.ErrNotExist && v.expectErr != err) { t.Errorf("expected %v, got %v, in %v", v.expectErr, err, i) t.Logf("%v, in %v", migrations, i) @@ -1285,7 +1298,7 @@ func TestReadDown(t *testing.T) { go m.readDown(v.from, v.limit, ret) migrations, err := migrationsFromChannel(ret) - if (v.expectErr == os.ErrNotExist && !os.IsNotExist(err)) || + if (v.expectErr == os.ErrNotExist && !errors.Is(err, os.ErrNotExist)) || (v.expectErr != os.ErrNotExist && v.expectErr != err) { t.Errorf("expected %v, got %v, in %v", v.expectErr, err, i) t.Logf("%v, in %v", migrations, i) @@ -1310,12 +1323,12 @@ func TestLock(t *testing.T) { func migrationsFromChannel(ret chan interface{}) ([]*Migration, error) { slice := make([]*Migration, 0) for r := range ret { - switch r.(type) { + switch t := r.(type) { case error: - return slice, r.(error) + return slice, t case *Migration: - slice = append(slice, r.(*Migration)) + slice = append(slice, t) } } return slice, nil @@ -1327,7 +1340,7 @@ func newMigSeq(migr ...*Migration) migrationSequence { return migr } -func (m *migrationSequence) add(migr ...*Migration) migrationSequence { +func (m *migrationSequence) add(migr ...*Migration) migrationSequence { // nolint:unused *m = append(*m, migr...) return *m } @@ -1375,7 +1388,7 @@ func M(version uint, targetVersion ...int) *Migration { // mr is a convenience func to create a new *Migration from the raw database query func mr(value string) *Migration { return &Migration{ - Body: ioutil.NopCloser(bytes.NewReader([]byte(value))), + Body: ioutil.NopCloser(strings.NewReader(value)), } } diff --git a/migration.go b/migration.go index 069e7f038..704fef49e 100644 --- a/migration.go +++ b/migration.go @@ -129,7 +129,9 @@ func (m *Migration) Buffer() error { // start reading from body, peek won't move the read pointer though // poor man's solution? - b.Peek(int(m.BufferSize)) + if _, err := b.Peek(int(m.BufferSize)); err != nil && err != io.EOF { + return err + } m.FinishedBuffering = time.Now() @@ -145,10 +147,14 @@ func (m *Migration) Buffer() error { // close bufferWriter so Buffer knows that there is no // more data coming - m.bufferWriter.Close() + if err := m.bufferWriter.Close(); err != nil { + return err + } // it's safe to close the Body too - m.Body.Close() + if err := m.Body.Close(); err != nil { + return err + } return nil } diff --git a/source/aws_s3/s3.go b/source/aws_s3/s3.go index 4af57bb2b..cec87ae35 100644 --- a/source/aws_s3/s3.go +++ b/source/aws_s3/s3.go @@ -21,37 +21,64 @@ func init() { type s3Driver struct { s3client s3iface.S3API - bucket string - prefix string + config *Config migrations *source.Migrations } +type Config struct { + Bucket string + Prefix string +} + func (s *s3Driver) Open(folder string) (source.Driver, error) { - u, err := url.Parse(folder) + config, err := parseURI(folder) if err != nil { return nil, err } + sess, err := session.NewSession() if err != nil { return nil, err } - driver := s3Driver{ - bucket: u.Host, - prefix: strings.Trim(u.Path, "/") + "/", - s3client: s3.New(sess), + + return WithInstance(s3.New(sess), config) +} + +func WithInstance(s3client s3iface.S3API, config *Config) (source.Driver, error) { + driver := &s3Driver{ + config: config, + s3client: s3client, migrations: source.NewMigrations(), } - err = driver.loadMigrations() + + if err := driver.loadMigrations(); err != nil { + return nil, err + } + + return driver, nil +} + +func parseURI(uri string) (*Config, error) { + u, err := url.Parse(uri) if err != nil { return nil, err } - return &driver, nil + + prefix := strings.Trim(u.Path, "/") + if prefix != "" { + prefix += "/" + } + + return &Config{ + Bucket: u.Host, + Prefix: prefix, + }, nil } func (s *s3Driver) loadMigrations() error { output, err := s.s3client.ListObjects(&s3.ListObjectsInput{ - Bucket: aws.String(s.bucket), - Prefix: aws.String(s.prefix), + Bucket: aws.String(s.config.Bucket), + Prefix: aws.String(s.config.Prefix), Delimiter: aws.String("/"), }) if err != nil { @@ -113,9 +140,9 @@ func (s *s3Driver) ReadDown(version uint) (io.ReadCloser, string, error) { } func (s *s3Driver) open(m *source.Migration) (io.ReadCloser, string, error) { - key := path.Join(s.prefix, m.Raw) + key := path.Join(s.config.Prefix, m.Raw) object, err := s.s3client.GetObject(&s3.GetObjectInput{ - Bucket: aws.String(s.bucket), + Bucket: aws.String(s.config.Bucket), Key: aws.String(key), }) if err != nil { diff --git a/source/aws_s3/s3_test.go b/source/aws_s3/s3_test.go index e103d509b..348c71806 100644 --- a/source/aws_s3/s3_test.go +++ b/source/aws_s3/s3_test.go @@ -8,8 +8,8 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" - "github.com/golang-migrate/migrate/v4/source" st "github.com/golang-migrate/migrate/v4/source/testing" + "github.com/stretchr/testify/assert" ) func Test(t *testing.T) { @@ -30,17 +30,62 @@ func Test(t *testing.T) { "prod/migrations/0-random-stuff/whatever.txt": "", }, } - driver := s3Driver{ - bucket: "some-bucket", - prefix: "prod/migrations/", - migrations: source.NewMigrations(), - s3client: &s3Client, - } - err := driver.loadMigrations() + driver, err := WithInstance(&s3Client, &Config{ + Bucket: "some-bucket", + Prefix: "prod/migrations/", + }) if err != nil { t.Fatal(err) } - st.Test(t, &driver) + st.Test(t, driver) +} + +func TestParseURI(t *testing.T) { + tests := []struct { + name string + uri string + config *Config + }{ + { + "with prefix, no trailing slash", + "s3://migration-bucket/production", + &Config{ + Bucket: "migration-bucket", + Prefix: "production/", + }, + }, + { + "without prefix, no trailing slash", + "s3://migration-bucket", + &Config{ + Bucket: "migration-bucket", + }, + }, + { + "with prefix, trailing slash", + "s3://migration-bucket/production/", + &Config{ + Bucket: "migration-bucket", + Prefix: "production/", + }, + }, + { + "without prefix, trailing slash", + "s3://migration-bucket/", + &Config{ + Bucket: "migration-bucket", + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + actual, err := parseURI(test.uri) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, test.config, actual) + }) + } } type fakeS3 struct { diff --git a/source/bitbucket/.gitignore b/source/bitbucket/.gitignore new file mode 100644 index 000000000..466bdfbbb --- /dev/null +++ b/source/bitbucket/.gitignore @@ -0,0 +1 @@ +.bitbucket_test_secrets diff --git a/source/bitbucket/README.md b/source/bitbucket/README.md new file mode 100644 index 000000000..ad0b77ba1 --- /dev/null +++ b/source/bitbucket/README.md @@ -0,0 +1,14 @@ +# bitbucket + +This driver is catered for those that want to source migrations from bitbucket cloud(https://bitbucket.com). + +`bitbucket://user:password@owner/repo/path#ref` + +| URL Query | WithInstance Config | Description | +|------------|---------------------|-------------| +| user | | The username of the user connecting | +| password | | User's password or an app password with repo read permission | +| owner | | the repo owner | +| repo | | the name of the repository | +| path | | path in repo to migrations | +| ref | | (optional) can be a SHA, branch, or tag | diff --git a/source/bitbucket/bitbucket.go b/source/bitbucket/bitbucket.go new file mode 100644 index 000000000..b448825c7 --- /dev/null +++ b/source/bitbucket/bitbucket.go @@ -0,0 +1,208 @@ +package bitbucket + +import ( + "fmt" + "io" + "io/ioutil" + nurl "net/url" + "os" + "path" + "path/filepath" + "strings" + + "github.com/golang-migrate/migrate/v4/source" + "github.com/ktrysmt/go-bitbucket" +) + +func init() { + source.Register("bitbucket", &Bitbucket{}) +} + +var ( + ErrNoUserInfo = fmt.Errorf("no username:password provided") + ErrNoAccessToken = fmt.Errorf("no password/app password") + ErrInvalidRepo = fmt.Errorf("invalid repo") + ErrInvalidBitbucketClient = fmt.Errorf("expected *bitbucket.Client") + ErrNoDir = fmt.Errorf("no directory") +) + +type Bitbucket struct { + config *Config + client *bitbucket.Client + migrations *source.Migrations +} + +type Config struct { + Owner string + Repo string + Path string + Ref string +} + +func (b *Bitbucket) Open(url string) (source.Driver, error) { + u, err := nurl.Parse(url) + if err != nil { + return nil, err + } + + if u.User == nil { + return nil, ErrNoUserInfo + } + + password, ok := u.User.Password() + if !ok { + return nil, ErrNoAccessToken + } + + cl := bitbucket.NewBasicAuth(u.User.Username(), password) + + cfg := &Config{} + // set owner, repo and path in repo + cfg.Owner = u.Host + pe := strings.Split(strings.Trim(u.Path, "/"), "/") + if len(pe) < 1 { + return nil, ErrInvalidRepo + } + cfg.Repo = pe[0] + if len(pe) > 1 { + cfg.Path = strings.Join(pe[1:], "/") + } + cfg.Ref = u.Fragment + + bi, err := WithInstance(cl, cfg) + if err != nil { + return nil, err + } + + return bi, nil +} + +func WithInstance(client *bitbucket.Client, config *Config) (source.Driver, error) { + bi := &Bitbucket{ + client: client, + config: config, + migrations: source.NewMigrations(), + } + + if err := bi.readDirectory(); err != nil { + return nil, err + } + + return bi, nil +} + +func (b *Bitbucket) readDirectory() error { + b.ensureFields() + + fOpt := &bitbucket.RepositoryFilesOptions{ + Owner: b.config.Owner, + RepoSlug: b.config.Repo, + Ref: b.config.Ref, + Path: b.config.Path, + } + + dirContents, err := b.client.Repositories.Repository.ListFiles(fOpt) + + if err != nil { + return err + } + + for _, fi := range dirContents { + + m, err := source.DefaultParse(filepath.Base(fi.Path)) + if err != nil { + continue // ignore files that we can't parse + } + if !b.migrations.Append(m) { + return fmt.Errorf("unable to parse file %v", fi.Path) + } + } + + return nil +} + +func (b *Bitbucket) ensureFields() { + if b.config == nil { + b.config = &Config{} + } +} + +func (b *Bitbucket) Close() error { + return nil +} + +func (b *Bitbucket) First() (version uint, er error) { + b.ensureFields() + + if v, ok := b.migrations.First(); !ok { + return 0, &os.PathError{Op: "first", Path: b.config.Path, Err: os.ErrNotExist} + } else { + return v, nil + } +} + +func (b *Bitbucket) Prev(version uint) (prevVersion uint, err error) { + b.ensureFields() + + if v, ok := b.migrations.Prev(version); !ok { + return 0, &os.PathError{Op: fmt.Sprintf("prev for version %v", version), Path: b.config.Path, Err: os.ErrNotExist} + } else { + return v, nil + } +} + +func (b *Bitbucket) Next(version uint) (nextVersion uint, err error) { + b.ensureFields() + + if v, ok := b.migrations.Next(version); !ok { + return 0, &os.PathError{Op: fmt.Sprintf("next for version %v", version), Path: b.config.Path, Err: os.ErrNotExist} + } else { + return v, nil + } +} + +func (b *Bitbucket) ReadUp(version uint) (r io.ReadCloser, identifier string, err error) { + b.ensureFields() + + if m, ok := b.migrations.Up(version); ok { + fBlobOpt := &bitbucket.RepositoryBlobOptions{ + Owner: b.config.Owner, + RepoSlug: b.config.Repo, + Ref: b.config.Ref, + Path: path.Join(b.config.Path, m.Raw), + } + file, err := b.client.Repositories.Repository.GetFileBlob(fBlobOpt) + if err != nil { + return nil, "", err + } + if file != nil { + r := file.Content + return ioutil.NopCloser(strings.NewReader(string(r))), m.Identifier, nil + } + } + return nil, "", &os.PathError{Op: fmt.Sprintf("read version %v", version), Path: b.config.Path, Err: os.ErrNotExist} +} + +func (b *Bitbucket) ReadDown(version uint) (r io.ReadCloser, identifier string, err error) { + b.ensureFields() + + if m, ok := b.migrations.Down(version); ok { + fBlobOpt := &bitbucket.RepositoryBlobOptions{ + Owner: b.config.Owner, + RepoSlug: b.config.Repo, + Ref: b.config.Ref, + Path: path.Join(b.config.Path, m.Raw), + } + file, err := b.client.Repositories.Repository.GetFileBlob(fBlobOpt) + + if err != nil { + return nil, "", err + } + if file != nil { + r := file.Content + + return ioutil.NopCloser(strings.NewReader(string(r))), m.Identifier, nil + } + } + return nil, "", &os.PathError{Op: fmt.Sprintf("read version %v", version), Path: b.config.Path, Err: os.ErrNotExist} +} diff --git a/source/bitbucket/bitbucket_test.go b/source/bitbucket/bitbucket_test.go new file mode 100644 index 000000000..b37406c31 --- /dev/null +++ b/source/bitbucket/bitbucket_test.go @@ -0,0 +1,33 @@ +package bitbucket + +import ( + "bytes" + "io/ioutil" + "testing" + + st "github.com/golang-migrate/migrate/v4/source/testing" +) + +var BitbucketTestSecret = "" // username:password + +func init() { + secrets, err := ioutil.ReadFile(".bitbucket_test_secrets") + if err == nil { + BitbucketTestSecret = string(bytes.TrimSpace(secrets)[:]) + } +} + +func Test(t *testing.T) { + if len(BitbucketTestSecret) == 0 { + t.Skip("test requires .bitbucket_test_secrets") + } + + b := &Bitbucket{} + + d, err := b.Open("bitbucket://" + BitbucketTestSecret + "@abhishekbipp/test-migration/migrations/test#master") + if err != nil { + t.Fatal(err) + } + + st.Test(t, d) +} diff --git a/source/driver.go b/source/driver.go index 05f97adab..63e6393c3 100644 --- a/source/driver.go +++ b/source/driver.go @@ -87,7 +87,7 @@ func Open(url string) (Driver, error) { d, ok := drivers[u.Scheme] driversMu.RUnlock() if !ok { - return nil, fmt.Errorf("source driver: unknown driver %v (forgotten import?)", u.Scheme) + return nil, fmt.Errorf("source driver: unknown driver '%s' (forgotten import?)", u.Scheme) } return d.Open(url) diff --git a/source/errors.go b/source/errors.go new file mode 100644 index 000000000..93d66e0d4 --- /dev/null +++ b/source/errors.go @@ -0,0 +1,15 @@ +package source + +import "os" + +// ErrDuplicateMigration is an error type for reporting duplicate migration +// files. +type ErrDuplicateMigration struct { + Migration + os.FileInfo +} + +// Error implements error interface. +func (e ErrDuplicateMigration) Error() string { + return "duplicate migration file: " + e.Name() +} diff --git a/source/file/file.go b/source/file/file.go index ce43a498a..d8b21dffb 100644 --- a/source/file/file.go +++ b/source/file/file.go @@ -1,15 +1,12 @@ package file import ( - "fmt" - "io" - "io/ioutil" nurl "net/url" "os" - "path" "path/filepath" "github.com/golang-migrate/migrate/v4/source" + "github.com/golang-migrate/migrate/v4/source/iofs" ) func init() { @@ -17,26 +14,43 @@ func init() { } type File struct { - url string - path string - migrations *source.Migrations + iofs.PartialDriver + url string + path string } func (f *File) Open(url string) (source.Driver, error) { - u, err := nurl.Parse(url) + p, err := parseURL(url) if err != nil { return nil, err } + nf := &File{ + url: url, + path: p, + } + if err := nf.Init(os.DirFS(p), "."); err != nil { + return nil, err + } + return nf, nil +} +func parseURL(url string) (string, error) { + u, err := nurl.Parse(url) + if err != nil { + return "", err + } // concat host and path to restore full path // host might be `.` - p := u.Host + u.Path + p := u.Opaque + if len(p) == 0 { + p = u.Host + u.Path + } if len(p) == 0 { // default to current directory if no path wd, err := os.Getwd() if err != nil { - return nil, err + return "", err } p = wd @@ -44,84 +58,9 @@ func (f *File) Open(url string) (source.Driver, error) { // make path absolute if relative abs, err := filepath.Abs(p) if err != nil { - return nil, err + return "", err } p = abs } - - // scan directory - files, err := ioutil.ReadDir(p) - if err != nil { - return nil, err - } - - nf := &File{ - url: url, - path: p, - migrations: source.NewMigrations(), - } - - for _, fi := range files { - if !fi.IsDir() { - m, err := source.DefaultParse(fi.Name()) - if err != nil { - continue // ignore files that we can't parse - } - if !nf.migrations.Append(m) { - return nil, fmt.Errorf("unable to parse file %v", fi.Name()) - } - } - } - return nf, nil -} - -func (f *File) Close() error { - // nothing do to here - return nil -} - -func (f *File) First() (version uint, err error) { - if v, ok := f.migrations.First(); !ok { - return 0, &os.PathError{Op: "first", Path: f.path, Err: os.ErrNotExist} - } else { - return v, nil - } -} - -func (f *File) Prev(version uint) (prevVersion uint, err error) { - if v, ok := f.migrations.Prev(version); !ok { - return 0, &os.PathError{Op: fmt.Sprintf("prev for version %v", version), Path: f.path, Err: os.ErrNotExist} - } else { - return v, nil - } -} - -func (f *File) Next(version uint) (nextVersion uint, err error) { - if v, ok := f.migrations.Next(version); !ok { - return 0, &os.PathError{Op: fmt.Sprintf("next for version %v", version), Path: f.path, Err: os.ErrNotExist} - } else { - return v, nil - } -} - -func (f *File) ReadUp(version uint) (r io.ReadCloser, identifier string, err error) { - if m, ok := f.migrations.Up(version); ok { - r, err := os.Open(path.Join(f.path, m.Raw)) - if err != nil { - return nil, "", err - } - return r, m.Identifier, nil - } - return nil, "", &os.PathError{Op: fmt.Sprintf("read version %v", version), Path: f.path, Err: os.ErrNotExist} -} - -func (f *File) ReadDown(version uint) (r io.ReadCloser, identifier string, err error) { - if m, ok := f.migrations.Down(version); ok { - r, err := os.Open(path.Join(f.path, m.Raw)) - if err != nil { - return nil, "", err - } - return r, m.Identifier, nil - } - return nil, "", &os.PathError{Op: fmt.Sprintf("read version %v", version), Path: f.path, Err: os.ErrNotExist} + return p, nil } diff --git a/source/file/file_test.go b/source/file/file_test.go index c9cfe61cf..357316aaf 100644 --- a/source/file/file_test.go +++ b/source/file/file_test.go @@ -1,6 +1,7 @@ package file import ( + "errors" "fmt" "io/ioutil" "os" @@ -16,7 +17,11 @@ func Test(t *testing.T) { if err != nil { t.Fatal(err) } - defer os.RemoveAll(tmpDir) + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + t.Error(err) + } + }() // write files that meet driver test requirements mustWriteFile(t, tmpDir, "1_foobar.up.sql", "1 up") @@ -46,7 +51,11 @@ func TestOpen(t *testing.T) { if err != nil { t.Fatal(err) } - defer os.RemoveAll(tmpDir) + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + t.Error(err) + } + }() mustWriteFile(t, tmpDir, "1_foobar.up.sql", "") mustWriteFile(t, tmpDir, "1_foobar.down.sql", "") @@ -67,13 +76,22 @@ func TestOpenWithRelativePath(t *testing.T) { if err != nil { t.Fatal(err) } - defer os.RemoveAll(tmpDir) + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + t.Error(err) + } + }() wd, err := os.Getwd() if err != nil { t.Fatal(err) } - defer os.Chdir(wd) // rescue working dir after we are done + defer func() { + // rescue working dir after we are done + if err := os.Chdir(wd); err != nil { + t.Log(err) + } + }() if err := os.Chdir(tmpDir); err != nil { t.Fatal(err) @@ -130,7 +148,11 @@ func TestOpenWithDuplicateVersion(t *testing.T) { if err != nil { t.Fatal(err) } - defer os.RemoveAll(tmpDir) + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + t.Error(err) + } + }() mustWriteFile(t, tmpDir, "1_foo.up.sql", "") // 1 up mustWriteFile(t, tmpDir, "1_bar.up.sql", "") // 1 up @@ -147,7 +169,11 @@ func TestClose(t *testing.T) { if err != nil { t.Fatal(err) } - defer os.RemoveAll(tmpDir) + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + t.Error(err) + } + }() f := &File{} d, err := f.Open("file://" + tmpDir) @@ -182,24 +208,35 @@ func mustCreateBenchmarkDir(t *testing.B) (dir string) { func BenchmarkOpen(b *testing.B) { dir := mustCreateBenchmarkDir(b) - defer os.RemoveAll(dir) + defer func() { + if err := os.RemoveAll(dir); err != nil { + b.Error(err) + } + }() b.ResetTimer() for n := 0; n < b.N; n++ { f := &File{} - f.Open("file://" + dir) + _, err := f.Open("file://" + dir) + if err != nil { + b.Error(err) + } } b.StopTimer() } func BenchmarkNext(b *testing.B) { dir := mustCreateBenchmarkDir(b) - defer os.RemoveAll(dir) + defer func() { + if err := os.RemoveAll(dir); err != nil { + b.Error(err) + } + }() f := &File{} d, _ := f.Open("file://" + dir) b.ResetTimer() v, err := d.First() for n := 0; n < b.N; n++ { - for !os.IsNotExist(err) { + for !errors.Is(err, os.ErrNotExist) { v, err = d.Next(v) } } diff --git a/source/github/README.md b/source/github/README.md index b4fbc1a22..76197e64f 100644 --- a/source/github/README.md +++ b/source/github/README.md @@ -1,11 +1,15 @@ # github -`github://user:personal-access-token@owner/repo/path#ref` +This driver is catered for those that want to source migrations from [github.com](https://github.com). The URL scheme doesn't require a hostname, as it just simply defaults to `github.com`. + +Authenticated client: `github://user:personal-access-token@owner/repo/path#ref` + +Unauthenticated client: `github://owner/repo/path#ref` | URL Query | WithInstance Config | Description | |------------|---------------------|-------------| -| user | | The username of the user connecting | -| personal-access-token | | An access token from Github (https://github.com/settings/tokens) | +| user | | (optional) The username of the user connecting | +| personal-access-token | | (optional) An access token from GitHub (https://github.com/settings/tokens) | | owner | | the repo owner | | repo | | the name of the repository | | path | | path in repo to migrations | diff --git a/source/github/github.go b/source/github/github.go index ac6f86da9..bf04da937 100644 --- a/source/github/github.go +++ b/source/github/github.go @@ -1,18 +1,19 @@ package github import ( - "bytes" "context" "fmt" + "golang.org/x/oauth2" "io" "io/ioutil" + "net/http" nurl "net/url" "os" "path" "strings" "github.com/golang-migrate/migrate/v4/source" - "github.com/google/go-github/github" + "github.com/google/go-github/v39/github" ) func init() { @@ -28,17 +29,17 @@ var ( ) type Github struct { - client *github.Client - url string - - pathOwner string - pathRepo string - path string + config *Config + client *github.Client options *github.RepositoryContentGetOptions migrations *source.Migrations } type Config struct { + Owner string + Repo string + Path string + Ref string } func (g *Github) Open(url string) (source.Driver, error) { @@ -47,36 +48,37 @@ func (g *Github) Open(url string) (source.Driver, error) { return nil, err } - if u.User == nil { - return nil, ErrNoUserInfo - } - - password, ok := u.User.Password() - if !ok { - return nil, ErrNoUserInfo - } + // client defaults to http.DefaultClient + var client *http.Client + if u.User != nil { + password, ok := u.User.Password() + if !ok { + return nil, ErrNoUserInfo + } + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: password}, + ) + client = oauth2.NewClient(context.Background(), ts) - tr := &github.BasicAuthTransport{ - Username: u.User.Username(), - Password: password, } gn := &Github{ - client: github.NewClient(tr.Client()), - url: url, + client: github.NewClient(client), migrations: source.NewMigrations(), options: &github.RepositoryContentGetOptions{Ref: u.Fragment}, } + gn.ensureFields() + // set owner, repo and path in repo - gn.pathOwner = u.Host + gn.config.Owner = u.Host pe := strings.Split(strings.Trim(u.Path, "/"), "/") if len(pe) < 1 { return nil, ErrInvalidRepo } - gn.pathRepo = pe[0] + gn.config.Repo = pe[0] if len(pe) > 1 { - gn.path = strings.Join(pe[1:], "/") + gn.config.Path = strings.Join(pe[1:], "/") } if err := gn.readDirectory(); err != nil { @@ -89,16 +91,29 @@ func (g *Github) Open(url string) (source.Driver, error) { func WithInstance(client *github.Client, config *Config) (source.Driver, error) { gn := &Github{ client: client, + config: config, migrations: source.NewMigrations(), + options: &github.RepositoryContentGetOptions{Ref: config.Ref}, } + if err := gn.readDirectory(); err != nil { return nil, err } + return gn, nil } func (g *Github) readDirectory() error { - fileContent, dirContents, _, err := g.client.Repositories.GetContents(context.Background(), g.pathOwner, g.pathRepo, g.path, g.options) + g.ensureFields() + + fileContent, dirContents, _, err := g.client.Repositories.GetContents( + context.Background(), + g.config.Owner, + g.config.Repo, + g.config.Path, + g.options, + ) + if err != nil { return err } @@ -119,37 +134,58 @@ func (g *Github) readDirectory() error { return nil } +func (g *Github) ensureFields() { + if g.config == nil { + g.config = &Config{} + } +} + func (g *Github) Close() error { return nil } -func (g *Github) First() (version uint, er error) { +func (g *Github) First() (version uint, err error) { + g.ensureFields() + if v, ok := g.migrations.First(); !ok { - return 0, &os.PathError{"first", g.path, os.ErrNotExist} + return 0, &os.PathError{Op: "first", Path: g.config.Path, Err: os.ErrNotExist} } else { return v, nil } } func (g *Github) Prev(version uint) (prevVersion uint, err error) { + g.ensureFields() + if v, ok := g.migrations.Prev(version); !ok { - return 0, &os.PathError{fmt.Sprintf("prev for version %v", version), g.path, os.ErrNotExist} + return 0, &os.PathError{Op: fmt.Sprintf("prev for version %v", version), Path: g.config.Path, Err: os.ErrNotExist} } else { return v, nil } } func (g *Github) Next(version uint) (nextVersion uint, err error) { + g.ensureFields() + if v, ok := g.migrations.Next(version); !ok { - return 0, &os.PathError{fmt.Sprintf("next for version %v", version), g.path, os.ErrNotExist} + return 0, &os.PathError{Op: fmt.Sprintf("next for version %v", version), Path: g.config.Path, Err: os.ErrNotExist} } else { return v, nil } } func (g *Github) ReadUp(version uint) (r io.ReadCloser, identifier string, err error) { + g.ensureFields() + if m, ok := g.migrations.Up(version); ok { - file, _, _, err := g.client.Repositories.GetContents(context.Background(), g.pathOwner, g.pathRepo, path.Join(g.path, m.Raw), g.options) + file, _, _, err := g.client.Repositories.GetContents( + context.Background(), + g.config.Owner, + g.config.Repo, + path.Join(g.config.Path, m.Raw), + g.options, + ) + if err != nil { return nil, "", err } @@ -158,15 +194,24 @@ func (g *Github) ReadUp(version uint) (r io.ReadCloser, identifier string, err e if err != nil { return nil, "", err } - return ioutil.NopCloser(bytes.NewReader([]byte(r))), m.Identifier, nil + return ioutil.NopCloser(strings.NewReader(r)), m.Identifier, nil } } - return nil, "", &os.PathError{fmt.Sprintf("read version %v", version), g.path, os.ErrNotExist} + return nil, "", &os.PathError{Op: fmt.Sprintf("read version %v", version), Path: g.config.Path, Err: os.ErrNotExist} } func (g *Github) ReadDown(version uint) (r io.ReadCloser, identifier string, err error) { + g.ensureFields() + if m, ok := g.migrations.Down(version); ok { - file, _, _, err := g.client.Repositories.GetContents(context.Background(), g.pathOwner, g.pathRepo, path.Join(g.path, m.Raw), g.options) + file, _, _, err := g.client.Repositories.GetContents( + context.Background(), + g.config.Owner, + g.config.Repo, + path.Join(g.config.Path, m.Raw), + g.options, + ) + if err != nil { return nil, "", err } @@ -175,8 +220,8 @@ func (g *Github) ReadDown(version uint) (r io.ReadCloser, identifier string, err if err != nil { return nil, "", err } - return ioutil.NopCloser(bytes.NewReader([]byte(r))), m.Identifier, nil + return ioutil.NopCloser(strings.NewReader(r)), m.Identifier, nil } } - return nil, "", &os.PathError{fmt.Sprintf("read version %v", version), g.path, os.ErrNotExist} + return nil, "", &os.PathError{Op: fmt.Sprintf("read version %v", version), Path: g.config.Path, Err: os.ErrNotExist} } diff --git a/source/github/github_test.go b/source/github/github_test.go index 227db2c84..3bee0bfe9 100644 --- a/source/github/github_test.go +++ b/source/github/github_test.go @@ -2,10 +2,12 @@ package github import ( "bytes" + "fmt" "io/ioutil" "testing" st "github.com/golang-migrate/migrate/v4/source/testing" + "github.com/stretchr/testify/assert" ) var GithubTestSecret = "" // username:token @@ -30,3 +32,28 @@ func Test(t *testing.T) { st.Test(t, d) } + +func TestDefaultClient(t *testing.T) { + g := &Github{} + owner := "golang-migrate" + repo := "migrate" + path := "source/github/examples/migrations" + + url := fmt.Sprintf("github://%s/%s/%s", owner, repo, path) + d, err := g.Open(url) + if err != nil { + t.Fatal(err) + } + + ver, err := d.First() + if err != nil { + t.Fatal(err) + } + assert.Equal(t, uint(1085649617), ver) + + ver, err = d.Next(ver) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, uint(1185749658), ver) +} diff --git a/source/github_ee/.gitignore b/source/github_ee/.gitignore new file mode 100644 index 000000000..3006ad5eb --- /dev/null +++ b/source/github_ee/.gitignore @@ -0,0 +1 @@ +.github_test_secrets diff --git a/source/github_ee/README.md b/source/github_ee/README.md new file mode 100644 index 000000000..69ce8e2b2 --- /dev/null +++ b/source/github_ee/README.md @@ -0,0 +1,21 @@ +# github ee + +## GitHub Enterprise Edition + +This driver is catered for those who run GitHub Enterprise under private infrastructure. + +The below URL scheme illustrates how to source migration files from GitHub Enterprise. + +GitHub client for Go requires API and Uploads endpoint hosts in order to create an instance of GitHub Enterprise Client. We're making an assumption that the API and Uploads are available under `https://api.*` and `https://uploads.*` respectively. [GitHub Enterprise Installation Guide](https://help.github.com/en/enterprise/2.15/admin/installation/enabling-subdomain-isolation) recommends that you enable Subdomain isolation feature. + +`github-ee://user:personal-access-token@host/owner/repo/path?verify-tls=true#ref` + +| URL Query | WithInstance Config | Description | +|------------|---------------------|-------------| +| user | | The username of the user connecting | +| personal-access-token | | Personal access token from your GitHub Enterprise instance | +| owner | | the repo owner | +| repo | | the name of the repository | +| path | | path in repo to migrations | +| ref | | (optional) can be a SHA, branch, or tag | +| verify-tls | | (optional) defaults to `true`. This option sets `tls.Config.InsecureSkipVerify` accordingly | diff --git a/source/github_ee/github_ee.go b/source/github_ee/github_ee.go new file mode 100644 index 000000000..57e41b12e --- /dev/null +++ b/source/github_ee/github_ee.go @@ -0,0 +1,97 @@ +package github_ee + +import ( + "crypto/tls" + "fmt" + "net/http" + nurl "net/url" + "strconv" + "strings" + + "github.com/golang-migrate/migrate/v4/source" + gh "github.com/golang-migrate/migrate/v4/source/github" + + "github.com/google/go-github/v39/github" +) + +func init() { + source.Register("github-ee", &GithubEE{}) +} + +type GithubEE struct { + source.Driver +} + +func (g *GithubEE) Open(url string) (source.Driver, error) { + verifyTLS := true + + u, err := nurl.Parse(url) + if err != nil { + return nil, err + } + + if o := u.Query().Get("verify-tls"); o != "" { + verifyTLS = parseBool(o, verifyTLS) + } + + if u.User == nil { + return nil, gh.ErrNoUserInfo + } + + password, ok := u.User.Password() + if !ok { + return nil, gh.ErrNoUserInfo + } + + ghc, err := g.createGithubClient(u.Host, u.User.Username(), password, verifyTLS) + if err != nil { + return nil, err + } + + pe := strings.Split(strings.Trim(u.Path, "/"), "/") + + if len(pe) < 1 { + return nil, gh.ErrInvalidRepo + } + + cfg := &gh.Config{ + Owner: pe[0], + Repo: pe[1], + Ref: u.Fragment, + } + + if len(pe) > 2 { + cfg.Path = strings.Join(pe[2:], "/") + } + + i, err := gh.WithInstance(ghc, cfg) + if err != nil { + return nil, err + } + + return &GithubEE{Driver: i}, nil +} + +func (g *GithubEE) createGithubClient(host, username, password string, verifyTLS bool) (*github.Client, error) { + tr := &github.BasicAuthTransport{ + Username: username, + Password: password, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: !verifyTLS}, + }, + } + + apiHost := fmt.Sprintf("https://%s/api/v3", host) + uploadHost := fmt.Sprintf("https://uploads.%s", host) + + return github.NewEnterpriseClient(apiHost, uploadHost, tr.Client()) +} + +func parseBool(val string, fallback bool) bool { + b, err := strconv.ParseBool(val) + if err != nil { + return fallback + } + + return b +} diff --git a/source/github_ee/github_ee_test.go b/source/github_ee/github_ee_test.go new file mode 100644 index 000000000..3a8224912 --- /dev/null +++ b/source/github_ee/github_ee_test.go @@ -0,0 +1,44 @@ +package github_ee + +import ( + "net/http" + "net/http/httptest" + nurl "net/url" + "testing" +) + +func Test(t *testing.T) { + ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v3/repos/mattes/migrate_test_tmp/contents/test" { + w.WriteHeader(http.StatusNotFound) + return + } + + if ref := r.URL.Query().Get("ref"); ref != "452b8003e7" { + w.WriteHeader(http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + + _, err := w.Write([]byte("[]")) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + })) + defer ts.Close() + + u, err := nurl.Parse(ts.URL) + if err != nil { + t.Fatal(err) + } + + g := &GithubEE{} + _, err = g.Open("github-ee://foo:bar@" + u.Host + "/mattes/migrate_test_tmp/test?verify-tls=false#452b8003e7") + + if err != nil { + t.Fatal(err) + } +} diff --git a/source/gitlab/.gitignore b/source/gitlab/.gitignore new file mode 100644 index 000000000..e295794d3 --- /dev/null +++ b/source/gitlab/.gitignore @@ -0,0 +1 @@ +.gitlab_test_secrets diff --git a/source/gitlab/README.md b/source/gitlab/README.md new file mode 100644 index 000000000..c8f1cb1a5 --- /dev/null +++ b/source/gitlab/README.md @@ -0,0 +1,12 @@ +# gitlab + +`gitlab://user:personal-access-token@gitlab_url/project_id/path#ref` + +| URL Query | WithInstance Config | Description | +|------------|---------------------|-------------| +| user | | The username of the user connecting | +| personal-access-token | | An access token from Gitlab (https:///profile/personal_access_tokens) | +| gitlab_url | | url of the gitlab server | +| project_id | | id of the repository | +| path | | path in repo to migrations | +| ref | | (optional) can be a SHA, branch, or tag | diff --git a/source/gitlab/examples/migrations/1085649617_create_users_table.down.sql b/source/gitlab/examples/migrations/1085649617_create_users_table.down.sql new file mode 100644 index 000000000..c99ddcdc8 --- /dev/null +++ b/source/gitlab/examples/migrations/1085649617_create_users_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS users; diff --git a/source/gitlab/examples/migrations/1085649617_create_users_table.up.sql b/source/gitlab/examples/migrations/1085649617_create_users_table.up.sql new file mode 100644 index 000000000..92897dcab --- /dev/null +++ b/source/gitlab/examples/migrations/1085649617_create_users_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE users ( + user_id integer unique, + name varchar(40), + email varchar(40) +); diff --git a/source/gitlab/examples/migrations/1185749658_add_city_to_users.down.sql b/source/gitlab/examples/migrations/1185749658_add_city_to_users.down.sql new file mode 100644 index 000000000..940c60712 --- /dev/null +++ b/source/gitlab/examples/migrations/1185749658_add_city_to_users.down.sql @@ -0,0 +1 @@ +ALTER TABLE users DROP COLUMN IF EXISTS city; diff --git a/source/gitlab/examples/migrations/1185749658_add_city_to_users.up.sql b/source/gitlab/examples/migrations/1185749658_add_city_to_users.up.sql new file mode 100644 index 000000000..67823edc9 --- /dev/null +++ b/source/gitlab/examples/migrations/1185749658_add_city_to_users.up.sql @@ -0,0 +1,3 @@ +ALTER TABLE users ADD COLUMN city varchar(100); + + diff --git a/source/gitlab/examples/migrations/1285849751_add_index_on_user_emails.down.sql b/source/gitlab/examples/migrations/1285849751_add_index_on_user_emails.down.sql new file mode 100644 index 000000000..3e87dd229 --- /dev/null +++ b/source/gitlab/examples/migrations/1285849751_add_index_on_user_emails.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS users_email_index; diff --git a/source/gitlab/examples/migrations/1285849751_add_index_on_user_emails.up.sql b/source/gitlab/examples/migrations/1285849751_add_index_on_user_emails.up.sql new file mode 100644 index 000000000..fbeb4ab4e --- /dev/null +++ b/source/gitlab/examples/migrations/1285849751_add_index_on_user_emails.up.sql @@ -0,0 +1,3 @@ +CREATE UNIQUE INDEX CONCURRENTLY users_email_index ON users (email); + +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/source/gitlab/examples/migrations/1385949617_create_books_table.down.sql b/source/gitlab/examples/migrations/1385949617_create_books_table.down.sql new file mode 100644 index 000000000..1a0b1a214 --- /dev/null +++ b/source/gitlab/examples/migrations/1385949617_create_books_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS books; diff --git a/source/gitlab/examples/migrations/1385949617_create_books_table.up.sql b/source/gitlab/examples/migrations/1385949617_create_books_table.up.sql new file mode 100644 index 000000000..f1503b518 --- /dev/null +++ b/source/gitlab/examples/migrations/1385949617_create_books_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE books ( + user_id integer, + name varchar(40), + author varchar(40) +); diff --git a/source/gitlab/examples/migrations/1485949617_create_movies_table.down.sql b/source/gitlab/examples/migrations/1485949617_create_movies_table.down.sql new file mode 100644 index 000000000..3a5187689 --- /dev/null +++ b/source/gitlab/examples/migrations/1485949617_create_movies_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS movies; diff --git a/source/gitlab/examples/migrations/1485949617_create_movies_table.up.sql b/source/gitlab/examples/migrations/1485949617_create_movies_table.up.sql new file mode 100644 index 000000000..f0ef5943b --- /dev/null +++ b/source/gitlab/examples/migrations/1485949617_create_movies_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE movies ( + user_id integer, + name varchar(40), + director varchar(40) +); diff --git a/source/gitlab/examples/migrations/1585849751_just_a_comment.up.sql b/source/gitlab/examples/migrations/1585849751_just_a_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/source/gitlab/examples/migrations/1585849751_just_a_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/source/gitlab/examples/migrations/1685849751_another_comment.up.sql b/source/gitlab/examples/migrations/1685849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/source/gitlab/examples/migrations/1685849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/source/gitlab/examples/migrations/1785849751_another_comment.up.sql b/source/gitlab/examples/migrations/1785849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/source/gitlab/examples/migrations/1785849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/source/gitlab/examples/migrations/1885849751_another_comment.up.sql b/source/gitlab/examples/migrations/1885849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/source/gitlab/examples/migrations/1885849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/source/gitlab/gitlab.go b/source/gitlab/gitlab.go new file mode 100644 index 000000000..76dc0b5dd --- /dev/null +++ b/source/gitlab/gitlab.go @@ -0,0 +1,240 @@ +package gitlab + +import ( + "encoding/base64" + "fmt" + "io" + "io/ioutil" + "net/http" + nurl "net/url" + "os" + "strconv" + "strings" +) + +import ( + "github.com/golang-migrate/migrate/v4/source" + "github.com/xanzy/go-gitlab" +) + +func init() { + source.Register("gitlab", &Gitlab{}) +} + +const DefaultMaxItemsPerPage = 100 + +var ( + ErrNoUserInfo = fmt.Errorf("no username:token provided") + ErrNoAccessToken = fmt.Errorf("no access token") + ErrInvalidHost = fmt.Errorf("invalid host") + ErrInvalidProjectID = fmt.Errorf("invalid project id") + ErrInvalidResponse = fmt.Errorf("invalid response") +) + +type Gitlab struct { + client *gitlab.Client + url string + + projectID string + path string + listOptions *gitlab.ListTreeOptions + getOptions *gitlab.GetFileOptions + migrations *source.Migrations +} + +type Config struct { +} + +func (g *Gitlab) Open(url string) (source.Driver, error) { + u, err := nurl.Parse(url) + if err != nil { + return nil, err + } + + if u.User == nil { + return nil, ErrNoUserInfo + } + + password, ok := u.User.Password() + if !ok { + return nil, ErrNoAccessToken + } + + gn := &Gitlab{ + client: gitlab.NewClient(nil, password), + url: url, + migrations: source.NewMigrations(), + } + + if u.Host != "" { + uri := nurl.URL{ + Scheme: "https", + Host: u.Host, + } + + err = gn.client.SetBaseURL(uri.String()) + if err != nil { + return nil, ErrInvalidHost + } + } + + pe := strings.Split(strings.Trim(u.Path, "/"), "/") + if len(pe) < 1 { + return nil, ErrInvalidProjectID + } + gn.projectID = pe[0] + if len(pe) > 1 { + gn.path = strings.Join(pe[1:], "/") + } + + gn.listOptions = &gitlab.ListTreeOptions{ + Path: &gn.path, + Ref: &u.Fragment, + ListOptions: gitlab.ListOptions{ + PerPage: DefaultMaxItemsPerPage, + }, + } + + gn.getOptions = &gitlab.GetFileOptions{ + Ref: &u.Fragment, + } + + if err := gn.readDirectory(); err != nil { + return nil, err + } + + return gn, nil +} + +func WithInstance(client *gitlab.Client, config *Config) (source.Driver, error) { + gn := &Gitlab{ + client: client, + migrations: source.NewMigrations(), + } + if err := gn.readDirectory(); err != nil { + return nil, err + } + return gn, nil +} + +func (g *Gitlab) readDirectory() error { + var nodes []*gitlab.TreeNode + for { + n, response, err := g.client.Repositories.ListTree(g.projectID, g.listOptions) + if err != nil { + return err + } + + if response.StatusCode != http.StatusOK { + return ErrInvalidResponse + } + + nodes = append(nodes, n...) + if response.CurrentPage >= response.TotalPages { + break + } + g.listOptions.ListOptions.Page = response.NextPage + } + + for i := range nodes { + m, err := g.nodeToMigration(nodes[i]) + if err != nil { + continue + } + + if !g.migrations.Append(m) { + return fmt.Errorf("unable to parse file %v", nodes[i].Name) + } + } + + return nil +} + +func (g *Gitlab) nodeToMigration(node *gitlab.TreeNode) (*source.Migration, error) { + m := source.Regex.FindStringSubmatch(node.Name) + if len(m) == 5 { + versionUint64, err := strconv.ParseUint(m[1], 10, 64) + if err != nil { + return nil, err + } + return &source.Migration{ + Version: uint(versionUint64), + Identifier: m[2], + Direction: source.Direction(m[3]), + Raw: g.path + "/" + node.Name, + }, nil + } + return nil, source.ErrParse +} + +func (g *Gitlab) Close() error { + return nil +} + +func (g *Gitlab) First() (version uint, er error) { + if v, ok := g.migrations.First(); !ok { + return 0, &os.PathError{Op: "first", Path: g.path, Err: os.ErrNotExist} + } else { + return v, nil + } +} + +func (g *Gitlab) Prev(version uint) (prevVersion uint, err error) { + if v, ok := g.migrations.Prev(version); !ok { + return 0, &os.PathError{Op: fmt.Sprintf("prev for version %v", version), Path: g.path, Err: os.ErrNotExist} + } else { + return v, nil + } +} + +func (g *Gitlab) Next(version uint) (nextVersion uint, err error) { + if v, ok := g.migrations.Next(version); !ok { + return 0, &os.PathError{Op: fmt.Sprintf("next for version %v", version), Path: g.path, Err: os.ErrNotExist} + } else { + return v, nil + } +} + +func (g *Gitlab) ReadUp(version uint) (r io.ReadCloser, identifier string, err error) { + if m, ok := g.migrations.Up(version); ok { + f, response, err := g.client.RepositoryFiles.GetFile(g.projectID, m.Raw, g.getOptions) + if err != nil { + return nil, "", err + } + + if response.StatusCode != http.StatusOK { + return nil, "", ErrInvalidResponse + } + + content, err := base64.StdEncoding.DecodeString(f.Content) + if err != nil { + return nil, "", err + } + + return ioutil.NopCloser(strings.NewReader(string(content))), m.Identifier, nil + } + + return nil, "", &os.PathError{Op: fmt.Sprintf("read version %v", version), Path: g.path, Err: os.ErrNotExist} +} + +func (g *Gitlab) ReadDown(version uint) (r io.ReadCloser, identifier string, err error) { + if m, ok := g.migrations.Down(version); ok { + f, response, err := g.client.RepositoryFiles.GetFile(g.projectID, m.Raw, g.getOptions) + if err != nil { + return nil, "", err + } + + if response.StatusCode != http.StatusOK { + return nil, "", ErrInvalidResponse + } + + content, err := base64.StdEncoding.DecodeString(f.Content) + if err != nil { + return nil, "", err + } + + return ioutil.NopCloser(strings.NewReader(string(content))), m.Identifier, nil + } + + return nil, "", &os.PathError{Op: fmt.Sprintf("read version %v", version), Path: g.path, Err: os.ErrNotExist} +} diff --git a/source/gitlab/gitlab_test.go b/source/gitlab/gitlab_test.go new file mode 100644 index 000000000..04cadd0cc --- /dev/null +++ b/source/gitlab/gitlab_test.go @@ -0,0 +1,32 @@ +package gitlab + +import ( + "bytes" + "io/ioutil" + "testing" + + st "github.com/golang-migrate/migrate/v4/source/testing" +) + +var GitlabTestSecret = "" // username:token + +func init() { + secrets, err := ioutil.ReadFile(".gitlab_test_secrets") + if err == nil { + GitlabTestSecret = string(bytes.TrimSpace(secrets)[:]) + } +} + +func Test(t *testing.T) { + if len(GitlabTestSecret) == 0 { + t.Skip("test requires .gitlab_test_secrets") + } + + g := &Gitlab{} + d, err := g.Open("gitlab://" + GitlabTestSecret + "@gitlab.com/11197284/migrations") + if err != nil { + t.Fatal(err) + } + + st.Test(t, d) +} diff --git a/source/go_bindata/examples/migrations/bindata.go b/source/go_bindata/examples/migrations/bindata.go index 282d5ef54..d0a57e24e 100644 --- a/source/go_bindata/examples/migrations/bindata.go +++ b/source/go_bindata/examples/migrations/bindata.go @@ -204,9 +204,9 @@ func AssetNames() []string { // _bindata is a table, holding each asset generator, mapped to its name. var _bindata = map[string]func() (*asset, error){ "1085649617_create_users_table.down.sql": _1085649617_create_users_tableDownSql, - "1085649617_create_users_table.up.sql": _1085649617_create_users_tableUpSql, - "1185749658_add_city_to_users.down.sql": _1185749658_add_city_to_usersDownSql, - "1185749658_add_city_to_users.up.sql": _1185749658_add_city_to_usersUpSql, + "1085649617_create_users_table.up.sql": _1085649617_create_users_tableUpSql, + "1185749658_add_city_to_users.down.sql": _1185749658_add_city_to_usersDownSql, + "1185749658_add_city_to_users.up.sql": _1185749658_add_city_to_usersUpSql, } // AssetDir returns the file names below a certain @@ -248,11 +248,12 @@ type bintree struct { Func func() (*asset, error) Children map[string]*bintree } + var _bintree = &bintree{nil, map[string]*bintree{ "1085649617_create_users_table.down.sql": &bintree{_1085649617_create_users_tableDownSql, map[string]*bintree{}}, - "1085649617_create_users_table.up.sql": &bintree{_1085649617_create_users_tableUpSql, map[string]*bintree{}}, - "1185749658_add_city_to_users.down.sql": &bintree{_1185749658_add_city_to_usersDownSql, map[string]*bintree{}}, - "1185749658_add_city_to_users.up.sql": &bintree{_1185749658_add_city_to_usersUpSql, map[string]*bintree{}}, + "1085649617_create_users_table.up.sql": &bintree{_1085649617_create_users_tableUpSql, map[string]*bintree{}}, + "1185749658_add_city_to_users.down.sql": &bintree{_1185749658_add_city_to_usersDownSql, map[string]*bintree{}}, + "1185749658_add_city_to_users.up.sql": &bintree{_1185749658_add_city_to_usersUpSql, map[string]*bintree{}}, }} // RestoreAsset restores an asset under the given directory @@ -301,4 +302,3 @@ func _filePath(dir, name string) string { cannonicalName := strings.Replace(name, "\\", "/", -1) return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) } - diff --git a/source/go_bindata/go-bindata.go b/source/go_bindata/go-bindata.go index d94758e88..5bd7bd853 100644 --- a/source/go_bindata/go-bindata.go +++ b/source/go_bindata/go-bindata.go @@ -74,7 +74,7 @@ func (b *Bindata) Close() error { func (b *Bindata) First() (version uint, err error) { if v, ok := b.migrations.First(); !ok { - return 0, &os.PathError{"first", b.path, os.ErrNotExist} + return 0, &os.PathError{Op: "first", Path: b.path, Err: os.ErrNotExist} } else { return v, nil } @@ -82,7 +82,7 @@ func (b *Bindata) First() (version uint, err error) { func (b *Bindata) Prev(version uint) (prevVersion uint, err error) { if v, ok := b.migrations.Prev(version); !ok { - return 0, &os.PathError{fmt.Sprintf("prev for version %v", version), b.path, os.ErrNotExist} + return 0, &os.PathError{Op: fmt.Sprintf("prev for version %v", version), Path: b.path, Err: os.ErrNotExist} } else { return v, nil } @@ -90,7 +90,7 @@ func (b *Bindata) Prev(version uint) (prevVersion uint, err error) { func (b *Bindata) Next(version uint) (nextVersion uint, err error) { if v, ok := b.migrations.Next(version); !ok { - return 0, &os.PathError{fmt.Sprintf("next for version %v", version), b.path, os.ErrNotExist} + return 0, &os.PathError{Op: fmt.Sprintf("next for version %v", version), Path: b.path, Err: os.ErrNotExist} } else { return v, nil } @@ -104,7 +104,7 @@ func (b *Bindata) ReadUp(version uint) (r io.ReadCloser, identifier string, err } return ioutil.NopCloser(bytes.NewReader(body)), m.Identifier, nil } - return nil, "", &os.PathError{fmt.Sprintf("read version %v", version), b.path, os.ErrNotExist} + return nil, "", &os.PathError{Op: fmt.Sprintf("read version %v", version), Path: b.path, Err: os.ErrNotExist} } func (b *Bindata) ReadDown(version uint) (r io.ReadCloser, identifier string, err error) { @@ -115,5 +115,5 @@ func (b *Bindata) ReadDown(version uint) (r io.ReadCloser, identifier string, er } return ioutil.NopCloser(bytes.NewReader(body)), m.Identifier, nil } - return nil, "", &os.PathError{fmt.Sprintf("read version %v", version), b.path, os.ErrNotExist} + return nil, "", &os.PathError{Op: fmt.Sprintf("read version %v", version), Path: b.path, Err: os.ErrNotExist} } diff --git a/source/godoc_vfs/vfs.go b/source/godoc_vfs/vfs.go index c0e6b1671..b9d31eca7 100644 --- a/source/godoc_vfs/vfs.go +++ b/source/godoc_vfs/vfs.go @@ -1,4 +1,4 @@ -// Package vfs contains a driver that reads migrations from a virtual file +// Package godoc_vfs contains a driver that reads migrations from a virtual file // system. // // Implementations of the filesystem interface that read from zip files and @@ -7,15 +7,11 @@ package godoc_vfs import ( - "bytes" - "fmt" - "io" - "io/ioutil" - "os" - "path" - "github.com/golang-migrate/migrate/v4/source" + "github.com/golang-migrate/migrate/v4/source/httpfs" + "golang.org/x/tools/godoc/vfs" + vfs_httpfs "golang.org/x/tools/godoc/vfs/httpfs" ) func init() { @@ -25,9 +21,9 @@ func init() { // VFS is an implementation of driver that returns migrations from a virtual // file system. type VFS struct { - migrations *source.Migrations - fs vfs.FileSystem - path string + httpfs.PartialDriver + fs vfs.FileSystem + path string } // Open implements the source.Driver interface for VFS. @@ -48,90 +44,13 @@ func WithInstance(fs vfs.FileSystem, searchPath string) (source.Driver, error) { } bn := &VFS{ - fs: fs, - path: searchPath, - migrations: source.NewMigrations(), + fs: fs, + path: searchPath, } - files, err := fs.ReadDir(searchPath) - if err != nil { + if err := bn.Init(vfs_httpfs.New(fs), searchPath); err != nil { return nil, err } - for _, fi := range files { - m, err := source.DefaultParse(fi.Name()) - if err != nil { - continue // ignore files that we can't parse - } - - if !bn.migrations.Append(m) { - return nil, fmt.Errorf("unable to parse file %v", fi) - } - } - return bn, nil } - -// Close implements the source.Driver interface for VFS. -// It is a no-op and should not be used. -func (b *VFS) Close() error { - return nil -} - -// First returns the first migration verion found in the file system. -// If no version is available os.ErrNotExist is returned. -func (b *VFS) First() (version uint, err error) { - v, ok := b.migrations.First() - if !ok { - return 0, &os.PathError{"first", "://" + b.path, os.ErrNotExist} - } - return v, nil -} - -// Prev returns the previous version available to the driver. -// If no previous version is available os.ErrNotExist is returned. -func (b *VFS) Prev(version uint) (prevVersion uint, err error) { - v, ok := b.migrations.Prev(version) - if !ok { - return 0, &os.PathError{fmt.Sprintf("prev for version %v", version), "://" + b.path, os.ErrNotExist} - } - return v, nil -} - -// Prev returns the next version available to the driver. -// If no previous version is available os.ErrNotExist is returned. -func (b *VFS) Next(version uint) (nextVersion uint, err error) { - v, ok := b.migrations.Next(version) - if !ok { - return 0, &os.PathError{fmt.Sprintf("next for version %v", version), "://" + b.path, os.ErrNotExist} - } - return v, nil -} - -// ReadUp returns the up migration body and an identifier that helps with -// finding this migration in the source. -// If there is no up migration available for this version it returns -// os.ErrNotExist. -func (b *VFS) ReadUp(version uint) (r io.ReadCloser, identifier string, err error) { - if m, ok := b.migrations.Up(version); ok { - body, err := vfs.ReadFile(b.fs, path.Join(b.path, m.Raw)) - if err != nil { - return nil, "", err - } - return ioutil.NopCloser(bytes.NewReader(body)), m.Identifier, nil - } - return nil, "", &os.PathError{fmt.Sprintf("read version %v", version), "://" + b.path, os.ErrNotExist} -} - -// ReadDown returns the down migration body and an identifier that helps with -// finding this migration in the source. -func (b *VFS) ReadDown(version uint) (r io.ReadCloser, identifier string, err error) { - if m, ok := b.migrations.Down(version); ok { - body, err := vfs.ReadFile(b.fs, path.Join(b.path, m.Raw)) - if err != nil { - return nil, "", err - } - return ioutil.NopCloser(bytes.NewReader(body)), m.Identifier, nil - } - return nil, "", &os.PathError{fmt.Sprintf("read version %v", version), "://" + b.path, os.ErrNotExist} -} diff --git a/source/godoc_vfs/vfs_example_test.go b/source/godoc_vfs/vfs_example_test.go index 9a57290d2..ae178294d 100644 --- a/source/godoc_vfs/vfs_example_test.go +++ b/source/godoc_vfs/vfs_example_test.go @@ -26,5 +26,8 @@ func Example_mapfs() { if err != nil { panic("error creating the migrations") } - m.Up() + err = m.Up() + if err != nil { + panic("up failed") + } } diff --git a/source/godoc_vfs/vfs_test.go b/source/godoc_vfs/vfs_test.go index f49360b14..30bced1ed 100644 --- a/source/godoc_vfs/vfs_test.go +++ b/source/godoc_vfs/vfs_test.go @@ -34,5 +34,7 @@ func TestOpen(t *testing.T) { } }() b := &godoc_vfs.VFS{} - b.Open("") + if _, err := b.Open(""); err != nil { + t.Error(err) + } } diff --git a/source/google_cloud_storage/storage.go b/source/google_cloud_storage/storage.go index 7aca5969a..9ec3e7a71 100644 --- a/source/google_cloud_storage/storage.go +++ b/source/google_cloud_storage/storage.go @@ -9,8 +9,8 @@ import ( "strings" "cloud.google.com/go/storage" + "context" "github.com/golang-migrate/migrate/v4/source" - "golang.org/x/net/context" "google.golang.org/api/iterator" ) diff --git a/source/httpfs/README.md b/source/httpfs/README.md new file mode 100644 index 000000000..f29b89df5 --- /dev/null +++ b/source/httpfs/README.md @@ -0,0 +1,49 @@ +# httpfs + +## Usage + +This package could be used to create new migration source drivers that uses +`http.FileSystem` to read migration files. + +Struct `httpfs.PartialDriver` partly implements the `source.Driver` interface. It has all +the methods except for `Open()`. Embedding this struct and adding `Open()` method +allows users of this package to create new migration sources. Example: + +```go +struct mydriver { + httpfs.PartialDriver +} + +func (d *mydriver) Open(url string) (source.Driver, error) { + var fs http.FileSystem + var path string + var ds mydriver + + // acquire fs and path from url + // set-up ds if necessary + + if err := ds.Init(fs, path); err != nil { + return nil, err + } + return &ds, nil +} +``` + +This package also provides a simple `source.Driver` implementation that works +with `http.FileSystem` provided by the user of this package. It is created with +`httpfs.New()` call. + +Example of using `http.Dir()` to read migrations from `sql` directory: + +```go + src, err := httpfs.New(http.Dir("sql")) + if err != nil { + // do something + } + m, err := migrate.NewWithSourceInstance("httpfs", src, "database://url") + if err != nil { + // do something + } + err = m.Up() + ... +``` diff --git a/source/httpfs/driver.go b/source/httpfs/driver.go new file mode 100644 index 000000000..e0cdbaa00 --- /dev/null +++ b/source/httpfs/driver.go @@ -0,0 +1,31 @@ +package httpfs + +import ( + "errors" + "net/http" + + "github.com/golang-migrate/migrate/v4/source" +) + +// driver is a migration source driver for reading migrations from +// http.FileSystem instances. It implements source.Driver interface and can be +// used as a migration source for the main migrate library. +type driver struct { + PartialDriver +} + +// New creates a new migrate source driver from a http.FileSystem instance and a +// relative path to migration files within the virtual FS. +func New(fs http.FileSystem, path string) (source.Driver, error) { + var d driver + if err := d.Init(fs, path); err != nil { + return nil, err + } + return &d, nil +} + +// Open completes the implementetion of source.Driver interface. Other methods +// are implemented by the embedded PartialDriver struct. +func (d *driver) Open(url string) (source.Driver, error) { + return nil, errors.New("Open() cannot be called on the httpfs passthrough driver") +} diff --git a/source/httpfs/driver_test.go b/source/httpfs/driver_test.go new file mode 100644 index 000000000..d0cf786f6 --- /dev/null +++ b/source/httpfs/driver_test.go @@ -0,0 +1,42 @@ +package httpfs_test + +import ( + "net/http" + "testing" + + "github.com/golang-migrate/migrate/v4/source/httpfs" + st "github.com/golang-migrate/migrate/v4/source/testing" +) + +func TestNewOK(t *testing.T) { + d, err := httpfs.New(http.Dir("testdata"), "sql") + if err != nil { + t.Errorf("New() expected not error, got: %s", err) + } + st.Test(t, d) +} + +func TestNewErrors(t *testing.T) { + d, err := httpfs.New(http.Dir("does-not-exist"), "") + if err == nil { + t.Errorf("New() expected to return error") + } + if d != nil { + t.Errorf("New() expected to return nil driver") + } +} + +func TestOpen(t *testing.T) { + d, err := httpfs.New(http.Dir("testdata/sql"), "") + if err != nil { + t.Error("New() expected no error") + return + } + d, err = d.Open("") + if d != nil { + t.Error("Open() expected to return nil driver") + } + if err == nil { + t.Error("Open() expected to return error") + } +} diff --git a/source/httpfs/partial_driver.go b/source/httpfs/partial_driver.go new file mode 100644 index 000000000..5ddb79883 --- /dev/null +++ b/source/httpfs/partial_driver.go @@ -0,0 +1,156 @@ +package httpfs + +import ( + "errors" + "io" + "net/http" + "os" + "path" + "strconv" + + "github.com/golang-migrate/migrate/v4/source" +) + +// PartialDriver is a helper service for creating new source drivers working with +// http.FileSystem instances. It implements all source.Driver interface methods +// except for Open(). New driver could embed this struct and add missing Open() +// method. +// +// To prepare PartialDriver for use Init() function. +type PartialDriver struct { + migrations *source.Migrations + fs http.FileSystem + path string +} + +// Init prepares not initialized PartialDriver instance to read migrations from a +// http.FileSystem instance and a relative path. +func (p *PartialDriver) Init(fs http.FileSystem, path string) error { + root, err := fs.Open(path) + if err != nil { + return err + } + + files, err := root.Readdir(0) + if err != nil { + _ = root.Close() + return err + } + if err = root.Close(); err != nil { + return err + } + + ms := source.NewMigrations() + for _, file := range files { + if file.IsDir() { + continue + } + + m, err := source.DefaultParse(file.Name()) + if err != nil { + continue // ignore files that we can't parse + } + + if !ms.Append(m) { + return source.ErrDuplicateMigration{ + Migration: *m, + FileInfo: file, + } + } + } + + p.fs = fs + p.path = path + p.migrations = ms + return nil +} + +// Close is part of source.Driver interface implementation. This is a no-op. +func (p *PartialDriver) Close() error { + return nil +} + +// First is part of source.Driver interface implementation. +func (p *PartialDriver) First() (version uint, err error) { + if version, ok := p.migrations.First(); ok { + return version, nil + } + return 0, &os.PathError{ + Op: "first", + Path: p.path, + Err: os.ErrNotExist, + } +} + +// Prev is part of source.Driver interface implementation. +func (p *PartialDriver) Prev(version uint) (prevVersion uint, err error) { + if version, ok := p.migrations.Prev(version); ok { + return version, nil + } + return 0, &os.PathError{ + Op: "prev for version " + strconv.FormatUint(uint64(version), 10), + Path: p.path, + Err: os.ErrNotExist, + } +} + +// Next is part of source.Driver interface implementation. +func (p *PartialDriver) Next(version uint) (nextVersion uint, err error) { + if version, ok := p.migrations.Next(version); ok { + return version, nil + } + return 0, &os.PathError{ + Op: "next for version " + strconv.FormatUint(uint64(version), 10), + Path: p.path, + Err: os.ErrNotExist, + } +} + +// ReadUp is part of source.Driver interface implementation. +func (p *PartialDriver) ReadUp(version uint) (r io.ReadCloser, identifier string, err error) { + if m, ok := p.migrations.Up(version); ok { + body, err := p.open(path.Join(p.path, m.Raw)) + if err != nil { + return nil, "", err + } + return body, m.Identifier, nil + } + return nil, "", &os.PathError{ + Op: "read up for version " + strconv.FormatUint(uint64(version), 10), + Path: p.path, + Err: os.ErrNotExist, + } +} + +// ReadDown is part of source.Driver interface implementation. +func (p *PartialDriver) ReadDown(version uint) (r io.ReadCloser, identifier string, err error) { + if m, ok := p.migrations.Down(version); ok { + body, err := p.open(path.Join(p.path, m.Raw)) + if err != nil { + return nil, "", err + } + return body, m.Identifier, nil + } + return nil, "", &os.PathError{ + Op: "read down for version " + strconv.FormatUint(uint64(version), 10), + Path: p.path, + Err: os.ErrNotExist, + } +} + +func (p *PartialDriver) open(path string) (http.File, error) { + f, err := p.fs.Open(path) + if err == nil { + return f, nil + } + // Some non-standard file systems may return errors that don't include the path, that + // makes debugging harder. + if !errors.As(err, new(*os.PathError)) { + err = &os.PathError{ + Op: "open", + Path: path, + Err: err, + } + } + return nil, err +} diff --git a/source/httpfs/partial_driver_test.go b/source/httpfs/partial_driver_test.go new file mode 100644 index 000000000..94c6ed14a --- /dev/null +++ b/source/httpfs/partial_driver_test.go @@ -0,0 +1,107 @@ +package httpfs_test + +import ( + "errors" + "net/http" + "strings" + "testing" + + "github.com/golang-migrate/migrate/v4/source" + "github.com/golang-migrate/migrate/v4/source/httpfs" + st "github.com/golang-migrate/migrate/v4/source/testing" +) + +type driver struct{ httpfs.PartialDriver } + +func (d *driver) Open(url string) (source.Driver, error) { return nil, errors.New("X") } + +type driverExample struct { + httpfs.PartialDriver +} + +func (d *driverExample) Open(url string) (source.Driver, error) { + parts := strings.Split(url, ":") + dir := parts[0] + path := "" + if len(parts) >= 2 { + path = parts[1] + } + + var de driverExample + return &de, de.Init(http.Dir(dir), path) +} + +func TestDriverExample(t *testing.T) { + d, err := (*driverExample)(nil).Open("testdata:sql") + if err != nil { + t.Errorf("Open() returned error: %s", err) + } + st.Test(t, d) +} + +func TestPartialDriverInit(t *testing.T) { + tests := []struct { + name string + fs http.FileSystem + path string + ok bool + }{ + { + name: "valid dir and empty path", + fs: http.Dir("testdata/sql"), + ok: true, + }, + { + name: "valid dir and non-empty path", + fs: http.Dir("testdata"), + path: "sql", + ok: true, + }, + { + name: "invalid dir", + fs: http.Dir("does-not-exist"), + }, + { + name: "file instead of dir", + fs: http.Dir("testdata/sql/1_foobar.up.sql"), + }, + { + name: "dir with duplicates", + fs: http.Dir("testdata/duplicates"), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var d driver + err := d.Init(test.fs, test.path) + if test.ok { + if err != nil { + t.Errorf("Init() returned error %s", err) + } + st.Test(t, &d) + if err = d.Close(); err != nil { + t.Errorf("Init().Close() returned error %s", err) + } + } else { + if err == nil { + t.Errorf("Init() expected error but did not get one") + } + } + }) + } + +} + +func TestFirstWithNoMigrations(t *testing.T) { + var d driver + fs := http.Dir("testdata/no-migrations") + + if err := d.Init(fs, ""); err != nil { + t.Errorf("No error on Init() expected, got: %v", err) + } + + if _, err := d.First(); err == nil { + t.Errorf("Expected error on First(), got: %v", err) + } +} diff --git a/source/httpfs/testdata/duplicates/1_foobar.up.sql b/source/httpfs/testdata/duplicates/1_foobar.up.sql new file mode 100644 index 000000000..046fd5a5d --- /dev/null +++ b/source/httpfs/testdata/duplicates/1_foobar.up.sql @@ -0,0 +1 @@ +1 up diff --git a/source/httpfs/testdata/duplicates/1_foobaz.up.sql b/source/httpfs/testdata/duplicates/1_foobaz.up.sql new file mode 100644 index 000000000..046fd5a5d --- /dev/null +++ b/source/httpfs/testdata/duplicates/1_foobaz.up.sql @@ -0,0 +1 @@ +1 up diff --git a/source/httpfs/testdata/no-migrations/some-file b/source/httpfs/testdata/no-migrations/some-file new file mode 100644 index 000000000..e69de29bb diff --git a/source/httpfs/testdata/sql/1_foobar.down.sql b/source/httpfs/testdata/sql/1_foobar.down.sql new file mode 100644 index 000000000..4267951a5 --- /dev/null +++ b/source/httpfs/testdata/sql/1_foobar.down.sql @@ -0,0 +1 @@ +1 down diff --git a/source/httpfs/testdata/sql/1_foobar.up.sql b/source/httpfs/testdata/sql/1_foobar.up.sql new file mode 100644 index 000000000..046fd5a5d --- /dev/null +++ b/source/httpfs/testdata/sql/1_foobar.up.sql @@ -0,0 +1 @@ +1 up diff --git a/source/httpfs/testdata/sql/3_foobar.up.sql b/source/httpfs/testdata/sql/3_foobar.up.sql new file mode 100644 index 000000000..77c1b77dc --- /dev/null +++ b/source/httpfs/testdata/sql/3_foobar.up.sql @@ -0,0 +1 @@ +3 up diff --git a/source/httpfs/testdata/sql/4_foobar.down.sql b/source/httpfs/testdata/sql/4_foobar.down.sql new file mode 100644 index 000000000..b405d8bd0 --- /dev/null +++ b/source/httpfs/testdata/sql/4_foobar.down.sql @@ -0,0 +1 @@ +4 down diff --git a/source/httpfs/testdata/sql/4_foobar.up.sql b/source/httpfs/testdata/sql/4_foobar.up.sql new file mode 100644 index 000000000..eba61bb94 --- /dev/null +++ b/source/httpfs/testdata/sql/4_foobar.up.sql @@ -0,0 +1 @@ +4 up diff --git a/source/httpfs/testdata/sql/5_foobar.down.sql b/source/httpfs/testdata/sql/5_foobar.down.sql new file mode 100644 index 000000000..6dc96e206 --- /dev/null +++ b/source/httpfs/testdata/sql/5_foobar.down.sql @@ -0,0 +1 @@ +5 down diff --git a/source/httpfs/testdata/sql/7_foobar.down.sql b/source/httpfs/testdata/sql/7_foobar.down.sql new file mode 100644 index 000000000..46636016b --- /dev/null +++ b/source/httpfs/testdata/sql/7_foobar.down.sql @@ -0,0 +1 @@ +7 down diff --git a/source/httpfs/testdata/sql/7_foobar.up.sql b/source/httpfs/testdata/sql/7_foobar.up.sql new file mode 100644 index 000000000..cdbc410ee --- /dev/null +++ b/source/httpfs/testdata/sql/7_foobar.up.sql @@ -0,0 +1 @@ +7 up diff --git a/source/httpfs/testdata/sql/other-files-are-ignored b/source/httpfs/testdata/sql/other-files-are-ignored new file mode 100644 index 000000000..e69de29bb diff --git a/source/httpfs/testdata/sql/subdirs-are-ignored/some-file b/source/httpfs/testdata/sql/subdirs-are-ignored/some-file new file mode 100644 index 000000000..e69de29bb diff --git a/source/iofs/README.md b/source/iofs/README.md new file mode 100644 index 000000000..d75b328b9 --- /dev/null +++ b/source/iofs/README.md @@ -0,0 +1,3 @@ +# iofs + +https://pkg.go.dev/github.com/golang-migrate/migrate/v4/source/iofs diff --git a/source/iofs/doc.go b/source/iofs/doc.go new file mode 100644 index 000000000..6b2c862e0 --- /dev/null +++ b/source/iofs/doc.go @@ -0,0 +1,10 @@ +/* +Package iofs provides the Go 1.16+ io/fs#FS driver. + +It can accept various file systems (like embed.FS, archive/zip#Reader) implementing io/fs#FS. + +This driver cannot be used with Go versions 1.15 and below. + +Also, Opening with a URL scheme is not supported. +*/ +package iofs diff --git a/source/iofs/example_test.go b/source/iofs/example_test.go new file mode 100644 index 000000000..474fc633c --- /dev/null +++ b/source/iofs/example_test.go @@ -0,0 +1,32 @@ +//go:build go1.16 +// +build go1.16 + +package iofs_test + +import ( + "embed" + "log" + + "github.com/golang-migrate/migrate/v4" + _ "github.com/golang-migrate/migrate/v4/database/postgres" + "github.com/golang-migrate/migrate/v4/source/iofs" +) + +//go:embed testdata/migrations/*.sql +var fs embed.FS + +func Example() { + d, err := iofs.New(fs, "testdata/migrations") + if err != nil { + log.Fatal(err) + } + m, err := migrate.NewWithSourceInstance("iofs", d, "postgres://postgres@localhost/postgres?sslmode=disable") + if err != nil { + log.Fatal(err) + } + err = m.Up() + if err != nil { + // ... + } + // ... +} diff --git a/source/iofs/iofs.go b/source/iofs/iofs.go new file mode 100644 index 000000000..dc934a5fe --- /dev/null +++ b/source/iofs/iofs.go @@ -0,0 +1,176 @@ +//go:build go1.16 +// +build go1.16 + +package iofs + +import ( + "errors" + "fmt" + "io" + "io/fs" + "path" + "strconv" + + "github.com/golang-migrate/migrate/v4/source" +) + +type driver struct { + PartialDriver +} + +// New returns a new Driver from io/fs#FS and a relative path. +func New(fsys fs.FS, path string) (source.Driver, error) { + var i driver + if err := i.Init(fsys, path); err != nil { + return nil, fmt.Errorf("failed to init driver with path %s: %w", path, err) + } + return &i, nil +} + +// Open is part of source.Driver interface implementation. +// Open cannot be called on the iofs passthrough driver. +func (d *driver) Open(url string) (source.Driver, error) { + return nil, errors.New("Open() cannot be called on the iofs passthrough driver") +} + +// PartialDriver is a helper service for creating new source drivers working with +// io/fs.FS instances. It implements all source.Driver interface methods +// except for Open(). New driver could embed this struct and add missing Open() +// method. +// +// To prepare PartialDriver for use Init() function. +type PartialDriver struct { + migrations *source.Migrations + fsys fs.FS + path string +} + +// Init prepares not initialized IoFS instance to read migrations from a +// io/fs#FS instance and a relative path. +func (d *PartialDriver) Init(fsys fs.FS, path string) error { + entries, err := fs.ReadDir(fsys, path) + if err != nil { + return err + } + + ms := source.NewMigrations() + for _, e := range entries { + if e.IsDir() { + continue + } + m, err := source.DefaultParse(e.Name()) + if err != nil { + continue + } + file, err := e.Info() + if err != nil { + return err + } + if !ms.Append(m) { + return source.ErrDuplicateMigration{ + Migration: *m, + FileInfo: file, + } + } + } + + d.fsys = fsys + d.path = path + d.migrations = ms + return nil +} + +// Close is part of source.Driver interface implementation. +// Closes the file system if possible. +func (d *PartialDriver) Close() error { + c, ok := d.fsys.(io.Closer) + if !ok { + return nil + } + return c.Close() +} + +// First is part of source.Driver interface implementation. +func (d *PartialDriver) First() (version uint, err error) { + if version, ok := d.migrations.First(); ok { + return version, nil + } + return 0, &fs.PathError{ + Op: "first", + Path: d.path, + Err: fs.ErrNotExist, + } +} + +// Prev is part of source.Driver interface implementation. +func (d *PartialDriver) Prev(version uint) (prevVersion uint, err error) { + if version, ok := d.migrations.Prev(version); ok { + return version, nil + } + return 0, &fs.PathError{ + Op: "prev for version " + strconv.FormatUint(uint64(version), 10), + Path: d.path, + Err: fs.ErrNotExist, + } +} + +// Next is part of source.Driver interface implementation. +func (d *PartialDriver) Next(version uint) (nextVersion uint, err error) { + if version, ok := d.migrations.Next(version); ok { + return version, nil + } + return 0, &fs.PathError{ + Op: "next for version " + strconv.FormatUint(uint64(version), 10), + Path: d.path, + Err: fs.ErrNotExist, + } +} + +// ReadUp is part of source.Driver interface implementation. +func (d *PartialDriver) ReadUp(version uint) (r io.ReadCloser, identifier string, err error) { + if m, ok := d.migrations.Up(version); ok { + body, err := d.open(path.Join(d.path, m.Raw)) + if err != nil { + return nil, "", err + } + return body, m.Identifier, nil + } + return nil, "", &fs.PathError{ + Op: "read up for version " + strconv.FormatUint(uint64(version), 10), + Path: d.path, + Err: fs.ErrNotExist, + } +} + +// ReadDown is part of source.Driver interface implementation. +func (d *PartialDriver) ReadDown(version uint) (r io.ReadCloser, identifier string, err error) { + if m, ok := d.migrations.Down(version); ok { + body, err := d.open(path.Join(d.path, m.Raw)) + if err != nil { + return nil, "", err + } + return body, m.Identifier, nil + } + return nil, "", &fs.PathError{ + Op: "read down for version " + strconv.FormatUint(uint64(version), 10), + Path: d.path, + Err: fs.ErrNotExist, + } +} + +func (d *PartialDriver) open(path string) (fs.File, error) { + f, err := d.fsys.Open(path) + if err == nil { + return f, nil + } + // Some non-standard file systems may return errors that don't include the path, that + // makes debugging harder. + if !errors.As(err, new(*fs.PathError)) { + err = &fs.PathError{ + Op: "open", + Path: path, + Err: err, + } + } + return nil, err +} diff --git a/source/iofs/iofs_test.go b/source/iofs/iofs_test.go new file mode 100644 index 000000000..d5b0ea019 --- /dev/null +++ b/source/iofs/iofs_test.go @@ -0,0 +1,21 @@ +//go:build go1.16 +// +build go1.16 + +package iofs_test + +import ( + "testing" + + "github.com/golang-migrate/migrate/v4/source/iofs" + st "github.com/golang-migrate/migrate/v4/source/testing" +) + +func Test(t *testing.T) { + // reuse the embed.FS set in example_test.go + d, err := iofs.New(fs, "testdata/migrations") + if err != nil { + t.Fatal(err) + } + + st.Test(t, d) +} diff --git a/source/iofs/testdata/migrations/1_foobar.down.sql b/source/iofs/testdata/migrations/1_foobar.down.sql new file mode 100644 index 000000000..4267951a5 --- /dev/null +++ b/source/iofs/testdata/migrations/1_foobar.down.sql @@ -0,0 +1 @@ +1 down diff --git a/source/iofs/testdata/migrations/1_foobar.up.sql b/source/iofs/testdata/migrations/1_foobar.up.sql new file mode 100644 index 000000000..046fd5a5d --- /dev/null +++ b/source/iofs/testdata/migrations/1_foobar.up.sql @@ -0,0 +1 @@ +1 up diff --git a/source/iofs/testdata/migrations/3_foobar.up.sql b/source/iofs/testdata/migrations/3_foobar.up.sql new file mode 100644 index 000000000..77c1b77dc --- /dev/null +++ b/source/iofs/testdata/migrations/3_foobar.up.sql @@ -0,0 +1 @@ +3 up diff --git a/source/iofs/testdata/migrations/4_foobar.down.sql b/source/iofs/testdata/migrations/4_foobar.down.sql new file mode 100644 index 000000000..b405d8bd0 --- /dev/null +++ b/source/iofs/testdata/migrations/4_foobar.down.sql @@ -0,0 +1 @@ +4 down diff --git a/source/iofs/testdata/migrations/4_foobar.up.sql b/source/iofs/testdata/migrations/4_foobar.up.sql new file mode 100644 index 000000000..eba61bb94 --- /dev/null +++ b/source/iofs/testdata/migrations/4_foobar.up.sql @@ -0,0 +1 @@ +4 up diff --git a/source/iofs/testdata/migrations/5_foobar.down.sql b/source/iofs/testdata/migrations/5_foobar.down.sql new file mode 100644 index 000000000..6dc96e206 --- /dev/null +++ b/source/iofs/testdata/migrations/5_foobar.down.sql @@ -0,0 +1 @@ +5 down diff --git a/source/iofs/testdata/migrations/7_foobar.down.sql b/source/iofs/testdata/migrations/7_foobar.down.sql new file mode 100644 index 000000000..46636016b --- /dev/null +++ b/source/iofs/testdata/migrations/7_foobar.down.sql @@ -0,0 +1 @@ +7 down diff --git a/source/iofs/testdata/migrations/7_foobar.up.sql b/source/iofs/testdata/migrations/7_foobar.up.sql new file mode 100644 index 000000000..cdbc410ee --- /dev/null +++ b/source/iofs/testdata/migrations/7_foobar.up.sql @@ -0,0 +1 @@ +7 up diff --git a/source/migration.go b/source/migration.go index fb94a331c..b8bb79020 100644 --- a/source/migration.go +++ b/source/migration.go @@ -67,7 +67,7 @@ func (i *Migrations) Append(m *Migration) (ok bool) { func (i *Migrations) buildIndex() { i.index = make(uintSlice, 0) - for version, _ := range i.migrations { + for version := range i.migrations { i.index = append(i.index, version) } sort.Sort(i.index) diff --git a/source/pkger/README.md b/source/pkger/README.md new file mode 100644 index 000000000..a4663df08 --- /dev/null +++ b/source/pkger/README.md @@ -0,0 +1,29 @@ +# pkger +```go +package main + +import ( + "errors" + "log" + + "github.com/golang-migrate/migrate/v4" + "github.com/markbates/pkger" + + _ "github.com/golang-migrate/migrate/v4/database/postgres" + _ "github.com/golang-migrate/migrate/v4/source/pkger" + _ "github.com/lib/pq" +) + +func main() { + pkger.Include("/module/path/to/migrations") + m, err := migrate.New("pkger:///module/path/to/migrations", "postgres://postgres@localhost/postgres?sslmode=disable") + if err != nil { + log.Fatalln(err) + } + if err := m.Up(); errors.Is(err, migrate.ErrNoChange) { + log.Println(err) + } else if err != nil { + log.Fatalln(err) + } +} +``` diff --git a/source/pkger/pkger.go b/source/pkger/pkger.go new file mode 100644 index 000000000..f5f2132d6 --- /dev/null +++ b/source/pkger/pkger.go @@ -0,0 +1,83 @@ +package pkger + +import ( + "fmt" + "net/http" + stdurl "net/url" + + "github.com/golang-migrate/migrate/v4/source" + "github.com/golang-migrate/migrate/v4/source/httpfs" + "github.com/markbates/pkger" + "github.com/markbates/pkger/pkging" +) + +func init() { + source.Register("pkger", &Pkger{}) +} + +// Pkger is a source.Driver that reads migrations from instances of +// pkging.Pkger. +type Pkger struct { + httpfs.PartialDriver +} + +// Open implements source.Driver. The path component of url will be used as the +// relative location of migrations. The returned driver will use the package +// scoped pkger.Open to access migrations. The relative root and any +// migrations must be added to the global pkger.Pkger instance by calling +// pkger.Apply. Refer to Pkger documentation for more information. +func (p *Pkger) Open(url string) (source.Driver, error) { + u, err := stdurl.Parse(url) + if err != nil { + return nil, err + } + + // wrap pkger to implement http.FileSystem. + fs := fsFunc(func(name string) (http.File, error) { + f, err := pkger.Open(name) + if err != nil { + return nil, err + } + return f.(http.File), nil + }) + + if err := p.Init(fs, u.Path); err != nil { + return nil, fmt.Errorf("failed to init driver with relative path %q: %w", u.Path, err) + } + + return p, nil +} + +// WithInstance returns a source.Driver that is backed by an instance of +// pkging.Pkger. The relative location of migrations is indicated by path. The +// path must exist on the pkging.Pkger instance for the driver to initialize +// successfully. +func WithInstance(instance pkging.Pkger, path string) (source.Driver, error) { + if instance == nil { + return nil, fmt.Errorf("expected instance of pkging.Pkger") + } + + // wrap pkger to implement http.FileSystem. + fs := fsFunc(func(name string) (http.File, error) { + f, err := instance.Open(name) + if err != nil { + return nil, err + } + return f.(http.File), nil + }) + + var p Pkger + + if err := p.Init(fs, path); err != nil { + return nil, fmt.Errorf("failed to init driver with relative path %q: %w", path, err) + } + + return &p, nil +} + +type fsFunc func(name string) (http.File, error) + +// Open implements http.FileSystem. +func (f fsFunc) Open(name string) (http.File, error) { + return f(name) +} diff --git a/source/pkger/pkger_test.go b/source/pkger/pkger_test.go new file mode 100644 index 000000000..bb3d561b6 --- /dev/null +++ b/source/pkger/pkger_test.go @@ -0,0 +1,196 @@ +package pkger + +import ( + "errors" + "os" + "testing" + + "github.com/gobuffalo/here" + st "github.com/golang-migrate/migrate/v4/source/testing" + "github.com/markbates/pkger" + "github.com/markbates/pkger/pkging" + "github.com/markbates/pkger/pkging/mem" +) + +func Test(t *testing.T) { + t.Run("WithInstance", func(t *testing.T) { + i := testInstance(t) + + createPkgerFile(t, i, "/1_foobar.up.sql") + createPkgerFile(t, i, "/1_foobar.down.sql") + createPkgerFile(t, i, "/3_foobar.up.sql") + createPkgerFile(t, i, "/4_foobar.up.sql") + createPkgerFile(t, i, "/4_foobar.down.sql") + createPkgerFile(t, i, "/5_foobar.down.sql") + createPkgerFile(t, i, "/7_foobar.up.sql") + createPkgerFile(t, i, "/7_foobar.down.sql") + + d, err := WithInstance(i, "/") + if err != nil { + t.Fatal(err) + } + + st.Test(t, d) + }) + + t.Run("Open", func(t *testing.T) { + i := testInstance(t) + + createPkgerFile(t, i, "/1_foobar.up.sql") + createPkgerFile(t, i, "/1_foobar.down.sql") + createPkgerFile(t, i, "/3_foobar.up.sql") + createPkgerFile(t, i, "/4_foobar.up.sql") + createPkgerFile(t, i, "/4_foobar.down.sql") + createPkgerFile(t, i, "/5_foobar.down.sql") + createPkgerFile(t, i, "/7_foobar.up.sql") + createPkgerFile(t, i, "/7_foobar.down.sql") + + registerPackageLevelInstance(t, i) + + d, err := (&Pkger{}).Open("pkger:///") + if err != nil { + t.Fatal(err) + } + + st.Test(t, d) + }) + +} + +func TestWithInstance(t *testing.T) { + t.Run("Subdir", func(t *testing.T) { + i := testInstance(t) + + // Make sure the relative root exists so that httpfs.PartialDriver can + // initialize. + createPkgerSubdir(t, i, "/subdir") + + _, err := WithInstance(i, "/subdir") + if err != nil { + t.Fatal("") + } + }) + + t.Run("NilInstance", func(t *testing.T) { + _, err := WithInstance(nil, "") + if err == nil { + t.Fatal(err) + } + }) + + t.Run("FailInit", func(t *testing.T) { + i := testInstance(t) + + _, err := WithInstance(i, "/fail") + if err == nil { + t.Fatal(err) + } + }) + + t.Run("FailWithoutMigrations", func(t *testing.T) { + i := testInstance(t) + + createPkgerSubdir(t, i, "/") + + d, err := WithInstance(i, "/") + if err != nil { + t.Fatal(err) + } + + if _, err := d.First(); !errors.Is(err, os.ErrNotExist) { + t.Fatal(err) + } + + }) +} + +func TestOpen(t *testing.T) { + + t.Run("InvalidURL", func(t *testing.T) { + _, err := (&Pkger{}).Open(":///") + if err == nil { + t.Fatal(err) + } + }) + + t.Run("Root", func(t *testing.T) { + _, err := (&Pkger{}).Open("pkger:///") + if err != nil { + t.Fatal(err) + } + }) + + t.Run("FailInit", func(t *testing.T) { + _, err := (&Pkger{}).Open("pkger:///subdir") + if err == nil { + t.Fatal(err) + } + }) + + i := testInstance(t) + createPkgerSubdir(t, i, "/subdir") + + // Note that this registers the instance globally so anything run after + // this will have access to everything container in the registered + // instance. + registerPackageLevelInstance(t, i) + + t.Run("Subdir", func(t *testing.T) { + _, err := (&Pkger{}).Open("pkger:///subdir") + if err != nil { + t.Fatal(err) + } + }) +} + +func TestClose(t *testing.T) { + d, err := (&Pkger{}).Open("pkger:///") + if err != nil { + t.Fatal(err) + } + if err := d.Close(); err != nil { + t.Fatal(err) + } +} + +func registerPackageLevelInstance(t *testing.T, pkg pkging.Pkger) { + if err := pkger.Apply(pkg, nil); err != nil { + t.Fatalf("failed to register pkger instance: %v\n", err) + } +} + +func testInstance(t *testing.T) pkging.Pkger { + pkg, err := inMemoryPkger() + if err != nil { + t.Fatalf("failed to create an pkging.Pkger instance: %v\n", err) + } + + return pkg +} + +func createPkgerSubdir(t *testing.T, pkg pkging.Pkger, subdir string) { + if err := pkg.MkdirAll(subdir, os.ModePerm); err != nil { + t.Fatalf("failed to create pkger subdir %q: %v\n", subdir, err) + } +} + +func createPkgerFile(t *testing.T, pkg pkging.Pkger, name string) { + _, err := pkg.Create(name) + if err != nil { + t.Fatalf("failed to create pkger file %q: %v\n", name, err) + } +} + +func inMemoryPkger() (*mem.Pkger, error) { + info, err := here.New().Current() + if err != nil { + return nil, err + } + + pkg, err := mem.New(info) + if err != nil { + return nil, err + } + + return pkg, nil +} diff --git a/source/stub/stub.go b/source/stub/stub.go index 839a686e1..a7776098f 100644 --- a/source/stub/stub.go +++ b/source/stub/stub.go @@ -48,7 +48,7 @@ func (s *Stub) Close() error { func (s *Stub) First() (version uint, err error) { if v, ok := s.Migrations.First(); !ok { - return 0, &os.PathError{"first", s.Url, os.ErrNotExist} // TODO: s.Url can be empty when called with WithInstance + return 0, &os.PathError{Op: "first", Path: s.Url, Err: os.ErrNotExist} // TODO: s.Url can be empty when called with WithInstance } else { return v, nil } @@ -56,7 +56,7 @@ func (s *Stub) First() (version uint, err error) { func (s *Stub) Prev(version uint) (prevVersion uint, err error) { if v, ok := s.Migrations.Prev(version); !ok { - return 0, &os.PathError{fmt.Sprintf("prev for version %v", version), s.Url, os.ErrNotExist} + return 0, &os.PathError{Op: fmt.Sprintf("prev for version %v", version), Path: s.Url, Err: os.ErrNotExist} } else { return v, nil } @@ -64,7 +64,7 @@ func (s *Stub) Prev(version uint) (prevVersion uint, err error) { func (s *Stub) Next(version uint) (nextVersion uint, err error) { if v, ok := s.Migrations.Next(version); !ok { - return 0, &os.PathError{fmt.Sprintf("next for version %v", version), s.Url, os.ErrNotExist} + return 0, &os.PathError{Op: fmt.Sprintf("next for version %v", version), Path: s.Url, Err: os.ErrNotExist} } else { return v, nil } @@ -74,12 +74,12 @@ func (s *Stub) ReadUp(version uint) (r io.ReadCloser, identifier string, err err if m, ok := s.Migrations.Up(version); ok { return ioutil.NopCloser(bytes.NewBufferString(m.Identifier)), fmt.Sprintf("%v.up.stub", version), nil } - return nil, "", &os.PathError{fmt.Sprintf("read up version %v", version), s.Url, os.ErrNotExist} + return nil, "", &os.PathError{Op: fmt.Sprintf("read up version %v", version), Path: s.Url, Err: os.ErrNotExist} } func (s *Stub) ReadDown(version uint) (r io.ReadCloser, identifier string, err error) { if m, ok := s.Migrations.Down(version); ok { return ioutil.NopCloser(bytes.NewBufferString(m.Identifier)), fmt.Sprintf("%v.down.stub", version), nil } - return nil, "", &os.PathError{fmt.Sprintf("read down version %v", version), s.Url, os.ErrNotExist} + return nil, "", &os.PathError{Op: fmt.Sprintf("read down version %v", version), Path: s.Url, Err: os.ErrNotExist} } diff --git a/source/testing/testing.go b/source/testing/testing.go index 0e2aa5269..6c148df1e 100644 --- a/source/testing/testing.go +++ b/source/testing/testing.go @@ -4,6 +4,7 @@ package testing import ( + "errors" "os" "testing" @@ -56,7 +57,7 @@ func TestPrev(t *testing.T, d source.Driver) { for i, v := range tt { pv, err := d.Prev(v.version) - if (v.expectErr == os.ErrNotExist && !os.IsNotExist(err)) && v.expectErr != err { + if (v.expectErr == os.ErrNotExist && !errors.Is(err, os.ErrNotExist)) && v.expectErr != err { t.Errorf("Prev: expected %v, got %v, in %v", v.expectErr, err, i) } if err == nil && v.expectPrevVersion != pv { @@ -85,7 +86,7 @@ func TestNext(t *testing.T, d source.Driver) { for i, v := range tt { nv, err := d.Next(v.version) - if (v.expectErr == os.ErrNotExist && !os.IsNotExist(err)) && v.expectErr != err { + if (v.expectErr == os.ErrNotExist && !errors.Is(err, os.ErrNotExist)) && v.expectErr != err { t.Errorf("Next: expected %v, got %v, in %v", v.expectErr, err, i) } if err == nil && v.expectNextVersion != nv { @@ -113,7 +114,7 @@ func TestReadUp(t *testing.T, d source.Driver) { for i, v := range tt { up, identifier, err := d.ReadUp(v.version) - if (v.expectErr == os.ErrNotExist && !os.IsNotExist(err)) || + if (v.expectErr == os.ErrNotExist && !errors.Is(err, os.ErrNotExist)) || (v.expectErr != os.ErrNotExist && err != v.expectErr) { t.Errorf("expected %v, got %v, in %v", v.expectErr, err, i) @@ -122,12 +123,17 @@ func TestReadUp(t *testing.T, d source.Driver) { t.Errorf("expected identifier not to be empty, in %v", i) } - if v.expectUp == true && up == nil { + if v.expectUp && up == nil { t.Errorf("expected up not to be nil, in %v", i) - } else if v.expectUp == false && up != nil { + } else if !v.expectUp && up != nil { t.Errorf("expected up to be nil, got %v, in %v", up, i) } } + if up != nil { + if err := up.Close(); err != nil { + t.Error(err) + } + } } } @@ -150,7 +156,7 @@ func TestReadDown(t *testing.T, d source.Driver) { for i, v := range tt { down, identifier, err := d.ReadDown(v.version) - if (v.expectErr == os.ErrNotExist && !os.IsNotExist(err)) || + if (v.expectErr == os.ErrNotExist && !errors.Is(err, os.ErrNotExist)) || (v.expectErr != os.ErrNotExist && err != v.expectErr) { t.Errorf("expected %v, got %v, in %v", v.expectErr, err, i) @@ -159,11 +165,16 @@ func TestReadDown(t *testing.T, d source.Driver) { t.Errorf("expected identifier not to be empty, in %v", i) } - if v.expectDown == true && down == nil { + if v.expectDown && down == nil { t.Errorf("expected down not to be nil, in %v", i) - } else if v.expectDown == false && down != nil { + } else if !v.expectDown && down != nil { t.Errorf("expected down to be nil, got %v, in %v", down, i) } } + if down != nil { + if err := down.Close(); err != nil { + t.Error(err) + } + } } } diff --git a/testing/docker.go b/testing/docker.go index 971d037dd..2a7c57f98 100644 --- a/testing/docker.go +++ b/testing/docker.go @@ -1,4 +1,6 @@ -// Package testing is used in driver tests. +// Package testing is used in driver tests and should only be used by migrate tests. +// +// Deprecated: If you'd like to test using Docker images, use package github.com/dhui/dktest instead package testing import ( @@ -11,6 +13,7 @@ import ( dockercontainer "github.com/docker/docker/api/types/container" dockernetwork "github.com/docker/docker/api/types/network" dockerclient "github.com/docker/docker/client" + "github.com/hashicorp/go-multierror" "io" "math/rand" "strconv" @@ -20,7 +23,10 @@ import ( ) func NewDockerContainer(t testing.TB, image string, env []string, cmd []string) (*DockerContainer, error) { - c, err := dockerclient.NewEnvClient() + c, err := dockerclient.NewClientWithOpts( + dockerclient.FromEnv, + dockerclient.WithAPIVersionNegotiation(), + ) if err != nil { return nil, err } @@ -62,7 +68,7 @@ type DockerContainer struct { keepForDebugging bool } -func (d *DockerContainer) PullImage() error { +func (d *DockerContainer) PullImage() (err error) { if d == nil { return errors.New("Cannot pull image on a nil *DockerContainer") } @@ -71,7 +77,11 @@ func (d *DockerContainer) PullImage() error { if err != nil { return err } - defer r.Close() + defer func() { + if errClose := r.Close(); errClose != nil { + err = multierror.Append(errClose) + } + }() // read output and log relevant lines bf := bufio.NewScanner(r) @@ -106,6 +116,7 @@ func (d *DockerContainer) Start() error { PublishAllPorts: true, }, &dockernetwork.NetworkingConfig{}, + nil, containerName) if err != nil { return err @@ -189,7 +200,7 @@ func (d *DockerContainer) Logs() (io.ReadCloser, error) { }) } -func (d *DockerContainer) portMapping(selectFirst bool, cPort int) (containerPort uint, hostIP string, hostPort uint, err error) { +func (d *DockerContainer) portMapping(selectFirst bool, cPort int) (containerPort uint, hostIP string, hostPort uint, err error) { // nolint:unparam if !d.containerInspected { if err := d.Inspect(); err != nil { d.t.Fatal(err) @@ -208,7 +219,7 @@ func (d *DockerContainer) portMapping(selectFirst bool, cPort int) (containerPor return 0, "", 0, err } - return uint(port.Int()), binding.HostIP, uint(hostPortUint), nil + return uint(port.Int()), binding.HostIP, uint(hostPortUint), nil // nolint: staticcheck } } diff --git a/testing/testing.go b/testing/testing.go index 579ae96ce..d7a2c2472 100644 --- a/testing/testing.go +++ b/testing/testing.go @@ -45,20 +45,25 @@ func ParallelTest(t *testing.T, versions []Version, readyFn IsReadyFunc, testFn } // make sure to remove container once done - defer container.Remove() - + defer func() { + if err := container.Remove(); err != nil { + t.Error(err) + } + }() // wait until database is ready - tick := time.Tick(1000 * time.Millisecond) - timeout := time.After(time.Duration(timeout) * time.Second) + tick := time.NewTicker(1000 * time.Millisecond) + defer tick.Stop() + timeout := time.NewTimer(time.Duration(timeout) * time.Second) + defer timeout.Stop() outer: for { select { - case <-tick: + case <-tick.C: if readyFn(container) { break outer } - case <-timeout: + case <-timeout.C: t.Fatalf("Docker: Container not ready, timeout for %v.\n%s", version, containerLogs(t, container)) } } @@ -76,7 +81,11 @@ func containerLogs(t *testing.T, c *DockerContainer) []byte { t.Error(err) return nil } - defer r.Close() + defer func() { + if err := r.Close(); err != nil { + t.Error(err) + } + }() b, err := ioutil.ReadAll(r) if err != nil { t.Error(err) diff --git a/testing/testing_test.go b/testing/testing_test.go index 8217decfa..d5e7ce1d0 100644 --- a/testing/testing_test.go +++ b/testing/testing_test.go @@ -4,7 +4,7 @@ import ( "testing" ) -func ExampleParallelTest(t *testing.T) { +func ExampleParallelTest(t *testing.T) { // nolint:govet var isReady = func(i Instance) bool { // Return true if Instance is ready to run tests. // Don't block here though. diff --git a/util.go b/util.go index 96b674669..26131a3ff 100644 --- a/util.go +++ b/util.go @@ -1,18 +1,22 @@ package migrate import ( - "errors" "fmt" nurl "net/url" "strings" ) // MultiError holds multiple errors. +// +// Deprecated: Use github.com/hashicorp/go-multierror instead type MultiError struct { Errs []error } // NewMultiError returns an error type holding multiple errors. +// +// Deprecated: Use github.com/hashicorp/go-multierror instead +// func NewMultiError(errs ...error) MultiError { compactErrs := make([]error, 0) for _, e := range errs { @@ -23,7 +27,7 @@ func NewMultiError(errs ...error) MultiError { return MultiError{compactErrs} } -// Error implements error. Mulitple errors are concatenated with 'and's. +// Error implements error. Multiple errors are concatenated with 'and's. func (m MultiError) Error() string { var strs = make([]string, 0) for _, e := range m.Errs { @@ -44,42 +48,6 @@ func suint(n int) uint { return uint(n) } -var errNoScheme = errors.New("no scheme") -var errEmptyURL = errors.New("URL cannot be empty") - -func sourceSchemeFromUrl(url string) (string, error) { - u, err := schemeFromUrl(url) - if err != nil { - return "", fmt.Errorf("source: %v", err) - } - return u, nil -} - -func databaseSchemeFromUrl(url string) (string, error) { - u, err := schemeFromUrl(url) - if err != nil { - return "", fmt.Errorf("database: %v", err) - } - return u, nil -} - -// schemeFromUrl returns the scheme from a URL string -func schemeFromUrl(url string) (string, error) { - if url == "" { - return "", errEmptyURL - } - - u, err := nurl.Parse(url) - if err != nil { - return "", err - } - if len(u.Scheme) == 0 { - return "", errNoScheme - } - - return u.Scheme, nil -} - // FilterCustomQuery filters all query values starting with `x-` func FilterCustomQuery(u *nurl.URL) *nurl.URL { ux := *u diff --git a/util_test.go b/util_test.go index b4843410b..1ad234473 100644 --- a/util_test.go +++ b/util_test.go @@ -1,7 +1,6 @@ package migrate import ( - "errors" nurl "net/url" "testing" ) @@ -31,85 +30,3 @@ func TestFilterCustomQuery(t *testing.T) { t.Fatalf("didn't expect x-custom") } } - -func TestSourceSchemeFromUrlSuccess(t *testing.T) { - urlStr := "protocol://path" - expected := "protocol" - - u, err := sourceSchemeFromUrl(urlStr) - if err != nil { - t.Fatalf("expected no error, but received %q", err) - } - if u != expected { - t.Fatalf("expected %q, but received %q", expected, u) - } -} - -func TestSourceSchemeFromUrlFailure(t *testing.T) { - cases := []struct { - name string - urlStr string - expectErr error - }{ - { - name: "Empty", - urlStr: "", - expectErr: errors.New("source: URL cannot be empty"), - }, - { - name: "NoScheme", - urlStr: "hello", - expectErr: errors.New("source: no scheme"), - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - _, err := sourceSchemeFromUrl(tc.urlStr) - if err.Error() != tc.expectErr.Error() { - t.Fatalf("expected %q, but received %q", tc.expectErr, err) - } - }) - } -} - -func TestDatabaseSchemeFromUrlSuccess(t *testing.T) { - urlStr := "protocol://path" - expected := "protocol" - - u, err := databaseSchemeFromUrl(urlStr) - if err != nil { - t.Fatalf("expected no error, but received %q", err) - } - if u != expected { - t.Fatalf("expected %q, but received %q", expected, u) - } -} - -func TestDatabaseSchemeFromUrlFailure(t *testing.T) { - cases := []struct { - name string - urlStr string - expectErr error - }{ - { - name: "Empty", - urlStr: "", - expectErr: errors.New("database: URL cannot be empty"), - }, - { - name: "NoScheme", - urlStr: "hello", - expectErr: errors.New("database: no scheme"), - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - _, err := databaseSchemeFromUrl(tc.urlStr) - if err.Error() != tc.expectErr.Error() { - t.Fatalf("expected %q, but received %q", tc.expectErr, err) - } - }) - } -}