Merge branch 'master' into 960_add_try_advisory_lock

.github/workflows/ci.yaml

@@ -9,24 +9,24 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
-          go-version: "1.20.x"
-      - uses: actions/checkout@v3
+          go-version: "1.22.x"
+      - uses: actions/checkout@v4
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@v4
         with:
           version: latest
 
   test:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: ["1.19.x", "1.20.x"]
+        go: ["1.21.x", "1.22.x"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go }}
 
@@ -60,30 +60,30 @@ jobs:
     # 3. When the workflow is triggered by a tag with `v` prefix
     if: ${{ success() && github.repository == 'golang-migrate/migrate' && startsWith(github.ref, 'refs/tags/v') }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: 2.7
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
-          go-version: "1.19.x"
+          go-version: "1.22.x"
 
-      - uses: docker/setup-qemu-action@v1
-      - uses: docker/setup-buildx-action@v1
-      - uses: docker/login-action@v1
+      - uses: docker/setup-qemu-action@v3
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
         with:
           username: golangmigrate
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - run: echo "SOURCE=$(make echo-source)" >> $GITHUB_ENV
       - run: echo "DATABASE=$(make echo-database)" >> $GITHUB_ENV
 
-      - uses: goreleaser/goreleaser-action@v2
+      - uses: goreleaser/goreleaser-action@v5
         with:
           version: latest
-          args: release --rm-dist
+          args: release --clean
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

.goreleaser.yml

@@ -87,7 +87,6 @@ release:
   prerelease: auto
 source:
   enabled: true
-  rlcp: true
   format: zip
 changelog:
   skip: false

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20-alpine3.18 AS builder
+FROM golang:1.22-alpine3.19 AS builder
 ARG VERSION
 
 RUN apk add --no-cache git gcc musl-dev make
@@ -15,7 +15,7 @@ COPY . ./
 
 RUN make build-docker
 
-FROM alpine:3.18
+FROM alpine:3.19
 
 RUN apk add --no-cache ca-certificates
 

Dockerfile.github-actions

@@ -1,4 +1,4 @@
-FROM alpine:3.18
+FROM alpine:3.19
 
 RUN apk add --no-cache ca-certificates
 

FAQ.md

@@ -50,7 +50,7 @@
   and whenever we want, not just once at the beginning of all tests.
 
 #### Can I maintain my driver in my own repository?
-  Yes, technically thats possible. We want to encourage you to contribute your driver to this respository though.
+  Yes, technically thats possible. We want to encourage you to contribute your driver to this repository though.
   The driver's functionality is dictated by migrate's interfaces. That means there should really
   just be one driver for a database/ source. We want to prevent a future where several drivers doing the exact same thing,
   just implemented a bit differently, co-exist somewhere on GitHub. If users have to do research first to find the

GETTING_STARTED.md

@@ -28,7 +28,7 @@ migrate -database YOUR_DATABASE_URL -path PATH_TO_YOUR_MIGRATIONS up
 
 Just add the code to your app and you're ready to go!
 
-Before commiting your migrations you should run your migrations up, down, and then up again to see if migrations are working properly both ways.
+Before committing your migrations you should run your migrations up, down, and then up again to see if migrations are working properly both ways.
 (e.g. if you created a table in a migration but reverse migration did not delete it, you will encounter an error when running the forward migration again)
 It's also worth checking your migrations in a separate, containerized environment. You can find some tools at the [end of this document](#further-reading).
 

Makefile

@@ -1,5 +1,5 @@
 SOURCE ?= file go_bindata github github_ee bitbucket aws_s3 google_cloud_storage godoc_vfs gitlab
-DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb yugabytedb clickhouse mongodb sqlserver firebird neo4j pgx pgx5
+DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb yugabytedb clickhouse mongodb sqlserver firebird neo4j pgx pgx5 rqlite
 DATABASE_TEST ?= $(DATABASE) sqlite sqlite3 sqlcipher
 VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-)
 TEST_FLAGS ?=

README.md

@@ -3,7 +3,7 @@
 [![Coverage Status](https://img.shields.io/coveralls/github/golang-migrate/migrate/master.svg)](https://coveralls.io/github/golang-migrate/migrate?branch=master)
 [![packagecloud.io](https://img.shields.io/badge/deb-packagecloud.io-844fec.svg)](https://packagecloud.io/golang-migrate/migrate?filter=debs)
 [![Docker Pulls](https://img.shields.io/docker/pulls/migrate/migrate.svg)](https://hub.docker.com/r/migrate/migrate/)
-![Supported Go Versions](https://img.shields.io/badge/Go-1.19%2C%201.20-lightgrey.svg)
+![Supported Go Versions](https://img.shields.io/badge/Go-1.21%2C%201.22-lightgrey.svg)
 [![GitHub Release](https://img.shields.io/github/release/golang-migrate/migrate.svg)](https://github.com/golang-migrate/migrate/releases)
 [![Go Report Card](https://goreportcard.com/badge/github.com/golang-migrate/migrate/v4)](https://goreportcard.com/report/github.com/golang-migrate/migrate/v4)
 
@@ -28,11 +28,11 @@ Database drivers run migrations. [Add a new database?](database/driver.go)
 * [PGX v5](database/pgx/v5)
 * [Redshift](database/redshift)
 * [Ql](database/ql)
-* [Cassandra](database/cassandra)
+* [Cassandra / ScyllaDB](database/cassandra)
 * [SQLite](database/sqlite)
 * [SQLite3](database/sqlite3) ([todo #165](https://github.com/mattes/migrate/issues/165))
 * [SQLCipher](database/sqlcipher)
-* [MySQL/ MariaDB](database/mysql)
+* [MySQL / MariaDB](database/mysql)
 * [Neo4j](database/neo4j)
 * [MongoDB](database/mongodb)
 * [CrateDB](database/crate) ([todo #170](https://github.com/mattes/migrate/issues/170))
@@ -43,6 +43,7 @@ Database drivers run migrations. [Add a new database?](database/driver.go)
 * [ClickHouse](database/clickhouse)
 * [Firebird](database/firebird)
 * [MS SQL Server](database/sqlserver)
+* [rqlite](database/rqlite)
 
 ### Database URLs
 
@@ -173,7 +174,7 @@ Each migration has an up and down migration. [Why?](FAQ.md#why-two-separate-file
 ## Coming from another db migration tool?
 
 Check out [migradaptor](https://github.com/musinit/migradaptor/).
-*Note: migradaptor is not affliated or supported by this project*
+*Note: migradaptor is not affiliated or supported by this project*
 
 ## Versions
 

database/cassandra/README.md

@@ -1,12 +1,17 @@
-# Cassandra
+# Cassandra / ScyllaDB
 
-* Drop command will not work on Cassandra 2.X because it rely on
+* `Drop()` method will not work on Cassandra 2.X because it rely on
 system_schema table which comes with 3.X
-* Other commands should work properly but are **not tested**
+* Other methods should work properly but are **not tested**
 * The Cassandra driver (gocql) does not natively support executing multiple statements in a single query. To allow for multiple statements in a single migration, you can use the `x-multi-statement` param. There are two important caveats:
   * This mode splits the migration text into separately-executed statements by a semi-colon `;`. Thus `x-multi-statement` cannot be used when a statement in the migration contains a string with a semi-colon.
   * The queries are not executed in any sort of transaction/batch, meaning you are responsible for fixing partial migrations.
 
+**ScyllaDB**
+
+* No additional configuration is required since it is a drop-in replacement for Cassandra.
+* The `Drop()` method` works for ScyllaDB 5.1
+
 
 ## Usage
 `cassandra://host:port/keyspace?param1=value&param2=value2`

database/cassandra/cassandra_test.go

@@ -25,9 +25,11 @@ var (
 	// Although Cassandra 2.x is supported by the Apache Foundation,
 	// the migrate db driver only supports Cassandra 3.x since it uses
 	// the system_schema keyspace.
+	// last ScyllaDB version tested is 5.1.11
 	specs = []dktesting.ContainerSpec{
 		{ImageName: "cassandra:3.0", Options: opts},
 		{ImageName: "cassandra:3.11", Options: opts},
+		{ImageName: "scylladb/scylla:5.1.11", Options: opts},
 	}
 )
 
@@ -59,6 +61,20 @@ func isReady(ctx context.Context, c dktest.ContainerInfo) bool {
 }
 
 func Test(t *testing.T) {
+	t.Run("test", test)
+	t.Run("testMigrate", testMigrate)
+
+	t.Cleanup(func() {
+		for _, spec := range specs {
+			t.Log("Cleaning up ", spec.ImageName)
+			if err := spec.Cleanup(); err != nil {
+				t.Error("Error removing ", spec.ImageName, "error:", err)
+			}
+		}
+	})
+}
+
+func test(t *testing.T) {
 	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
 		ip, port, err := c.Port(9042)
 		if err != nil {
@@ -79,7 +95,7 @@ func Test(t *testing.T) {
 	})
 }
 
-func TestMigrate(t *testing.T) {
+func testMigrate(t *testing.T) {
 	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
 		ip, port, err := c.Port(9042)
 		if err != nil {

database/clickhouse/README.md

@@ -16,10 +16,11 @@
 
 ## Notes
 
-* The Clickhouse driver does not natively support executing multipe statements in a single query. To allow for multiple statements in a single migration, you can use the `x-multi-statement` param. There are two important caveats:
+* The Clickhouse driver does not natively support executing multiple statements in a single query. To allow for multiple statements in a single migration, you can use the `x-multi-statement` param. There are two important caveats:
   * This mode splits the migration text into separately-executed statements by a semi-colon `;`. Thus `x-multi-statement` cannot be used when a statement in the migration contains a string with a semi-colon.
   * The queries are not executed in any sort of transaction/batch, meaning you are responsible for fixing partial migrations.
 * Using the default TinyLog table engine for the schema_versions table prevents backing up the table if using the [clickhouse-backup](https://github.com/AlexAkulov/clickhouse-backup) tool. If backing up the database with make sure the migrations are run with `x-migrations-table-engine=MergeTree`.
 * Clickhouse cluster mode is not officially supported, since it's not tested right now, but you can try enabling `schema_migrations` table replication by specifying a `x-cluster-name`:
   * When `x-cluster-name` is specified, `x-migrations-table-engine` also should be specified. See the docs regarding [replicated table engines](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/replication/#table_engines-replication).
   * When `x-cluster-name` is specified, only the `schema_migrations` table is replicated across the cluster. You still need to write your migrations so that the application tables are replicated within the cluster.
+* If you want to create database inside the migration, you should know, that table which will manage migrations `schema-migrations table` will be in `default` table, so you can't use `USE <database_name>` inside migration. In this case you may not specify the database in the connection string (example you can find [here](examples/migrations/003_create_database.up.sql))

database/clickhouse/clickhouse.go

@@ -220,7 +220,7 @@ func (ch *ClickHouse) ensureVersionTable() (err error) {
 
 	var (
 		table string
-		query = "SHOW TABLES FROM " + ch.config.DatabaseName + " LIKE '" + ch.config.MigrationsTable + "'"
+		query = "SHOW TABLES FROM " + quoteIdentifier(ch.config.DatabaseName) + " LIKE '" + ch.config.MigrationsTable + "'"
 	)
 	// check if migration table exists
 	if err := ch.conn.QueryRow(query).Scan(&table); err != nil {
@@ -259,7 +259,7 @@ func (ch *ClickHouse) ensureVersionTable() (err error) {
 }
 
 func (ch *ClickHouse) Drop() (err error) {
-	query := "SHOW TABLES FROM " + ch.config.DatabaseName
+	query := "SHOW TABLES FROM " + quoteIdentifier(ch.config.DatabaseName)
 	tables, err := ch.conn.Query(query)
 
 	if err != nil {
@@ -277,7 +277,7 @@ func (ch *ClickHouse) Drop() (err error) {
 			return err
 		}
 
-		query = "DROP TABLE IF EXISTS " + ch.config.DatabaseName + "." + table
+		query = "DROP TABLE IF EXISTS " + quoteIdentifier(ch.config.DatabaseName) + "." + quoteIdentifier(table)
 
 		if _, err := ch.conn.Exec(query); err != nil {
 			return &database.Error{OrigErr: err, Query: []byte(query)}
@@ -305,3 +305,12 @@ func (ch *ClickHouse) Unlock() error {
 	return nil
 }
 func (ch *ClickHouse) Close() error { return ch.conn.Close() }
+
+// Copied from lib/pq implementation: https://github.com/lib/pq/blob/v1.9.0/conn.go#L1611
+func quoteIdentifier(name string) string {
+	end := strings.IndexRune(name, 0)
+	if end > -1 {
+		name = name[:end]
+	}
+	return `"` + strings.Replace(name, `"`, `""`, -1) + `"`
+}

database/clickhouse/examples/migrations/003_create_database.down.sql

@@ -0,0 +1,10 @@
+DROP TABLE IF EXISTS driver_ratings;
+DROP TABLE IF EXISTS user_ratings;
+DROP TABLE IF EXISTS orders;
+DROP TABLE IF EXISTS driver_ratings_queue;
+DROP TABLE IF EXISTS user_ratings_queue;
+DROP TABLE IF EXISTS orders_queue;
+DROP VIEW IF EXISTS user_ratings_queue_mv;
+DROP VIEW IF EXISTS driver_ratings_queue_mv;
+DROP VIEW IF EXISTS orders_queue_mv;
+DROP DATABASE IF EXISTS analytics;

database/clickhouse/examples/migrations/003_create_database.up.sql

@@ -0,0 +1,81 @@
+CREATE DATABASE IF NOT EXISTS analytics;
+
+CREATE TABLE IF NOT EXISTS analytics.driver_ratings(
+    rate UInt8,
+    userID Int64,
+    driverID String,
+    orderID String,
+    inserted_time DateTime DEFAULT now()
+) ENGINE = MergeTree
+PARTITION BY driverID
+ORDER BY (inserted_time);
+
+CREATE TABLE analytics.driver_ratings_queue(
+    rate UInt8,
+    userID Int64,
+    driverID String,
+    orderID String
+) ENGINE = Kafka
+SETTINGS kafka_broker_list = 'broker:9092',
+    kafka_topic_list = 'driver-ratings',
+    kafka_group_name = 'rating_readers',
+    kafka_format = 'Avro',
+    kafka_max_block_size = 1048576;
+
+CREATE MATERIALIZED VIEW analytics.driver_ratings_queue_mv TO analytics.driver_ratings AS
+SELECT rate, userID, driverID, orderID
+FROM analytics.driver_ratings_queue;
+
+CREATE TABLE IF NOT EXISTS analytics.user_ratings(
+    rate UInt8,
+    userID Int64,
+    driverID String,
+    orderID String,
+    inserted_time DateTime DEFAULT now()
+) ENGINE = MergeTree
+    PARTITION BY userID
+    ORDER BY (inserted_time);
+
+CREATE TABLE analytics.user_ratings_queue(
+    rate UInt8,
+    userID Int64,
+    driverID String,
+    orderID String
+) ENGINE = Kafka
+SETTINGS kafka_broker_list = 'broker:9092',
+    kafka_topic_list = 'user-ratings',
+    kafka_group_name = 'rating_readers',
+    kafka_format = 'JSON',
+    kafka_max_block_size = 1048576;
+
+CREATE MATERIALIZED VIEW analytics.user_ratings_queue_mv TO analytics.user_ratings AS
+SELECT rate, userID, driverID, orderID
+FROM analytics.user_ratings_queue;
+
+CREATE TABLE IF NOT EXISTS analytics.orders(
+    from_place String,
+    to_place String,
+    userID Int64,
+    driverID String,
+    orderID String,
+    inserted_time DateTime DEFAULT now()
+) ENGINE = MergeTree
+    PARTITION BY driverID
+    ORDER BY (inserted_time);
+
+CREATE TABLE analytics.orders_queue(
+    from_place String,
+    to_place String,
+    userID Int64,
+    driverID String,
+    orderID String
+) ENGINE = Kafka
+SETTINGS kafka_broker_list = 'broker:9092',
+    kafka_topic_list = 'orders',
+    kafka_group_name = 'order_readers',
+    kafka_format = 'Avro',
+    kafka_max_block_size = 1048576;
+
+CREATE MATERIALIZED VIEW analytics.orders_queue_mv TO orders AS
+SELECT from_place, to_place, userID, driverID, orderID
+FROM analytics.orders_queue;

database/cockroachdb/TUTORIAL.md

@@ -15,7 +15,7 @@ CREATE USER IF NOT EXISTS cockroach;
 GRANT ALL ON DATABASE example TO cockroach;
 ```
 
-When using Migrate CLI we need to pass to database URL. Let's export it to a variable for convienience:
+When using Migrate CLI we need to pass to database URL. Let's export it to a variable for convenience:
 ```
 export COCKROACHDB_URL='cockroachdb://cockroach:@localhost:26257/example?sslmode=disable'
 ```