libsql database driver

Makefile

@@ -1,5 +1,5 @@
 SOURCE ?= file go_bindata github github_ee bitbucket aws_s3 google_cloud_storage godoc_vfs gitlab
-DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb yugabytedb clickhouse mongodb sqlserver firebird neo4j pgx pgx5
+DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb yugabytedb clickhouse mongodb sqlserver firebird neo4j pgx pgx5 libsql
 DATABASE_TEST ?= $(DATABASE) sqlite sqlite3 sqlcipher
 VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-)
 TEST_FLAGS ?=

database/libsql/README.md

@@ -0,0 +1,13 @@
+# libsql
+
+libsql implements the [libsql-client-go](https://github.com/libsql/libsql-client-go) interface for use with [migrate](https://github.com/golang-migrate/migrate).
+
+It can be used to connect to any database supported by libsql:
+
+- Local SQLite database files (See [Notes](#notes))
+- libSQL sqld instances (including Turso)
+
+## Notes
+
+- Uses the `github.com/libsql/libsql-client-go` libsql db driver (go)
+  - [No support for prepared statements using sqld with https](https://github.com/libsql/libsql-client-go/#compatibility-with-databasesql)

database/libsql/examples/migrations/33_create_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS pets;

database/libsql/examples/migrations/33_create_table.up.sql

@@ -0,0 +1,3 @@
+CREATE TABLE pets (
+  name string
+);

database/libsql/examples/migrations/44_alter_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS pets;

database/libsql/examples/migrations/44_alter_table.up.sql

@@ -0,0 +1 @@
+ALTER TABLE pets ADD predator bool;

database/libsql/libsql.go

@@ -0,0 +1,247 @@
+package libsql
+
+import (
+	"database/sql"
+	"fmt"
+	"io"
+
+	nurl "net/url"
+
+	"github.com/golang-migrate/migrate/v4"
+	"github.com/golang-migrate/migrate/v4/database"
+	"github.com/hashicorp/go-multierror"
+	"go.uber.org/atomic"
+
+	_ "github.com/libsql/libsql-client-go/libsql"
+	_ "modernc.org/sqlite"
+)
+
+func init() {
+	database.Register("libsql", &Libsql{})
+}
+
+var DefaultMigrationsTable = "schema_migrations"
+var (
+	ErrDatabaseDirty      = fmt.Errorf("database is dirty")
+	ErrNilConfig          = fmt.Errorf("no config")
+	ErrNoDatabaseName     = fmt.Errorf("no database name")
+	ErrDatabaseURLInvalid = fmt.Errorf("invalid database url")
+)
+
+type Config struct {
+	MigrationsTable string
+
+	DatabaseURL string
+}
+
+type Libsql struct {
+	db       *sql.DB
+	isLocked atomic.Bool
+
+	config *Config
+}
+
+var _ database.Driver = (*Libsql)(nil)
+
+func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) {
+	if config == nil {
+		return nil, ErrNilConfig
+	}
+
+	if err := instance.Ping(); err != nil {
+		return nil, err
+	}
+
+	if len(config.MigrationsTable) == 0 {
+		config.MigrationsTable = DefaultMigrationsTable
+	}
+
+	lx := &Libsql{
+		db:     instance,
+		config: config,
+	}
+	if err := lx.ensureVersionTable(); err != nil {
+		return nil, err
+	}
+	return lx, nil
+}
+
+func (l *Libsql) ensureVersionTable() (err error) {
+	if err = l.Lock(); err != nil {
+		return err
+	}
+
+	defer func() {
+		if e := l.Unlock(); e != nil {
+			if err == nil {
+				err = e
+			} else {
+				err = multierror.Append(err, e)
+			}
+		}
+	}()
+
+	query := fmt.Sprintf(`
+	CREATE TABLE IF NOT EXISTS %s (version uint64,dirty bool);
+  CREATE UNIQUE INDEX IF NOT EXISTS version_unique ON %s (version);
+  `, l.config.MigrationsTable, l.config.MigrationsTable)
+
+	if _, err := l.db.Exec(query); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Close implements database.Driver.
+func (l *Libsql) Close() error {
+	return l.db.Close()
+}
+
+func (l *Libsql) Drop() (err error) {
+	query := `SELECT name FROM sqlite_master WHERE type = 'table';`
+	tables, err := l.db.Query(query)
+	if err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+	defer func() {
+		if errClose := tables.Close(); errClose != nil {
+			err = multierror.Append(err, errClose)
+		}
+	}()
+
+	tableNames := make([]string, 0)
+	for tables.Next() {
+		var tableName string
+		if err := tables.Scan(&tableName); err != nil {
+			return err
+		}
+		if len(tableName) > 0 {
+			tableNames = append(tableNames, tableName)
+		}
+	}
+	if err := tables.Err(); err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+
+	if len(tableNames) > 0 {
+		for _, t := range tableNames {
+			query := "DROP TABLE " + t
+			err = l.executeQuery(query)
+			if err != nil {
+				return &database.Error{OrigErr: err, Query: []byte(query)}
+			}
+		}
+	}
+
+	return nil
+}
+
+func (l *Libsql) Lock() error {
+	if !l.isLocked.CAS(false, true) {
+		return database.ErrLocked
+	}
+	return nil
+}
+
+func (l *Libsql) Unlock() error {
+	if !l.isLocked.CAS(true, false) {
+		return database.ErrNotLocked
+	}
+	return nil
+}
+
+func (l *Libsql) Open(url string) (database.Driver, error) {
+	purl, err := nurl.Parse(url)
+	if err != nil {
+		return nil, err
+	}
+
+	dbUrl := migrate.FilterCustomQuery(purl).String()
+	db, err := sql.Open("libsql", dbUrl)
+	if err != nil {
+		return nil, err
+	}
+
+	qv := purl.Query()
+	migrationsTable := qv.Get("x-migrations-table")
+	if len(migrationsTable) == 0 {
+		migrationsTable = DefaultMigrationsTable
+	}
+
+	lx, err := WithInstance(db, &Config{
+		DatabaseURL:     dbUrl,
+		MigrationsTable: migrationsTable,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return lx, nil
+}
+
+func (l *Libsql) Run(migration io.Reader) error {
+	migr, err := io.ReadAll(migration)
+	if err != nil {
+		return err
+	}
+	query := string(migr[:])
+
+	return l.executeQuery(query)
+}
+
+func (l *Libsql) SetVersion(version int, dirty bool) error {
+	tx, err := l.db.Begin()
+	if err != nil {
+		return &database.Error{OrigErr: err, Err: "transaction start failed"}
+	}
+
+	query := "DELETE FROM " + l.config.MigrationsTable
+	if _, err := tx.Exec(query); err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+
+	// Also re-write the schema version for nil dirty versions to prevent
+	// empty schema version for failed down migration on the first migration
+	// See: https://github.com/golang-migrate/migrate/issues/330
+	if version >= 0 || (version == database.NilVersion && dirty) {
+		query := fmt.Sprintf(`INSERT INTO %s (version, dirty) VALUES (?, ?)`, l.config.MigrationsTable)
+		if _, err := tx.Exec(query, version, dirty); err != nil {
+			if errRollback := tx.Rollback(); errRollback != nil {
+				err = multierror.Append(err, errRollback)
+			}
+			return &database.Error{OrigErr: err, Query: []byte(query)}
+		}
+	}
+
+	if err := tx.Commit(); err != nil {
+		return &database.Error{OrigErr: err, Err: "transaction commit failed"}
+	}
+
+	return nil
+}
+
+func (l *Libsql) executeQuery(query string) error {
+	tx, err := l.db.Begin()
+	if err != nil {
+		return &database.Error{OrigErr: err, Err: "transaction start failed"}
+	}
+	if _, err := tx.Exec(query); err != nil {
+		if errRollback := tx.Rollback(); errRollback != nil {
+			err = multierror.Append(err, errRollback)
+		}
+		return &database.Error{OrigErr: err, Query: []byte(query)}
+	}
+	if err := tx.Commit(); err != nil {
+		return &database.Error{OrigErr: err, Err: "transaction commit failed"}
+	}
+	return nil
+}
+
+func (l *Libsql) Version() (version int, dirty bool, err error) {
+	query := "SELECT version, dirty FROM " + l.config.MigrationsTable + " LIMIT 1"
+	err = l.db.QueryRow(query).Scan(&version, &dirty)
+	if err != nil {
+		return database.NilVersion, false, nil
+	}
+	return version, dirty, nil
+}

database/libsql/libsql_test.go

@@ -0,0 +1,21 @@
+package libsql
+
+import (
+	"fmt"
+	"path/filepath"
+	"testing"
+
+	dt "github.com/golang-migrate/migrate/v4/database/testing"
+)
+
+func Test(t *testing.T) {
+	dir := t.TempDir()
+	t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite.db"))
+	l := &Libsql{}
+	addr := fmt.Sprintf("file:%s", filepath.Join(dir, "sqlite.db"))
+	d, err := l.Open(addr)
+	if err != nil {
+		t.Fatal(err)
+	}
+	dt.Test(t, d, []byte("CREATE TABLE t (Qty int, Name string);"))
+}

go.mod

@@ -38,7 +38,14 @@ require (
 	golang.org/x/tools v0.9.1
 	google.golang.org/api v0.126.0
 	modernc.org/ql v1.0.0
-	modernc.org/sqlite v1.18.1
+	modernc.org/sqlite v1.27.0
+)
+
+require (
+	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/libsql/sqlite-antlr4-parser v0.0.0-20230802215326-5cb5bb604475 // indirect
+	nhooyr.io/websocket v1.8.7 // indirect
 )
 
 require (
@@ -123,10 +130,11 @@ require (
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/klauspost/asmfmt v1.3.2 // indirect
-	github.com/klauspost/compress v1.15.11 // indirect
-	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/klauspost/compress v1.15.15 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.3 // indirect
+	github.com/libsql/libsql-client-go v0.0.0-20231026052543-fce76c0f39a7
 	github.com/mattn/go-colorable v0.1.6 // indirect
-	github.com/mattn/go-isatty v0.0.16 // indirect
+	github.com/mattn/go-isatty v0.0.18 // indirect
 	github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
 	github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
 	github.com/mitchellh/mapstructure v1.1.2 // indirect
@@ -141,7 +149,7 @@ require (
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/sirupsen/logrus v1.9.2 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
@@ -155,8 +163,8 @@ require (
 	golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 // indirect
 	golang.org/x/mod v0.10.0 // indirect
 	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/sync v0.2.0 // indirect
-	golang.org/x/sys v0.8.0 // indirect
+	golang.org/x/sync v0.3.0 // indirect
+	golang.org/x/sys v0.9.0 // indirect
 	golang.org/x/term v0.8.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
@@ -170,22 +178,24 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	lukechampine.com/uint128 v1.2.0 // indirect
 	modernc.org/b v1.0.0 // indirect
-	modernc.org/cc/v3 v3.36.3 // indirect
-	modernc.org/ccgo/v3 v3.16.9 // indirect
+	modernc.org/cc/v3 v3.40.0 // indirect
+	modernc.org/ccgo/v3 v3.16.13 // indirect
 	modernc.org/db v1.0.0 // indirect
 	modernc.org/file v1.0.0 // indirect
 	modernc.org/fileutil v1.0.0 // indirect
 	modernc.org/golex v1.0.0 // indirect
 	modernc.org/internal v1.0.0 // indirect
-	modernc.org/libc v1.17.1 // indirect
+	modernc.org/libc v1.29.0 // indirect
 	modernc.org/lldb v1.0.0 // indirect
-	modernc.org/mathutil v1.5.0 // indirect
-	modernc.org/memory v1.2.1 // indirect
+	modernc.org/mathutil v1.6.0 // indirect
+	modernc.org/memory v1.7.2 // indirect
 	modernc.org/opt v0.1.3 // indirect
 	modernc.org/sortutil v1.1.0 // indirect
 	modernc.org/strutil v1.1.3 // indirect
-	modernc.org/token v1.0.0 // indirect
+	modernc.org/token v1.0.1 // indirect
 	modernc.org/zappy v1.0.0 // indirect
 )
 
-go 1.18
+go 1.21
+
+toolchain go1.21.3