From 1d8534aa4dff5d140a7d2b6a94ef88d838311494 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 6 Sep 2022 10:02:38 -0400 Subject: [PATCH 01/35] Add Serialize and Deserialize support --- sqlite3.go | 44 ++++++++++++++++++++ sqlite3_test.go | 105 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+) diff --git a/sqlite3.go b/sqlite3.go index 5ac95709..cef691e8 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -905,6 +905,50 @@ func (c *SQLiteConn) begin(ctx context.Context) (driver.Tx, error) { return &SQLiteTx{c}, nil } +// Serialize returns a byte slice that is a serialization of the database. +// If the database fails to serialize, a nil slice will be returned. +// +// See https://www.sqlite.org/c3ref/serialize.html +func (c *SQLiteConn) Serialize(schema string) []byte { + if schema == "" { + schema = "main" + } + var zSchema *C.char + zSchema = C.CString(schema) + defer C.free(unsafe.Pointer(zSchema)) + + var sz C.sqlite3_int64 + ptr := C.sqlite3_serialize(c.db, zSchema, &sz, 0) + if ptr == nil { + return nil + } + defer C.sqlite3_free(unsafe.Pointer(ptr)) + return C.GoBytes(unsafe.Pointer(ptr), C.int(sz)) +} + +// Deserialize causes the connection to disconnect from the current database +// and then re-open as an in-memory database based on the contents of the +// byte slice. If deserelization fails, error will contain the return code +// of the underlying SQLite API call. +// +// See https://www.sqlite.org/c3ref/deserialize.html +func (c *SQLiteConn) Deserialize(b []byte, schema string) error { + if schema == "" { + schema = "main" + } + var zSchema *C.char + zSchema = C.CString(schema) + defer C.free(unsafe.Pointer(zSchema)) + + rc := C.sqlite3_deserialize(c.db, zSchema, + (*C.uint8_t)(unsafe.Pointer(&b[0])), + C.sqlite3_int64(len(b)), C.sqlite3_int64(len(b)), 0) + if rc != 0 { + return fmt.Errorf("deserialize failed with return %v", rc) + } + return nil +} + // Open database and return a new connection. // // A pragma can take either zero or one argument. diff --git a/sqlite3_test.go b/sqlite3_test.go index 878ec495..e5660c58 100644 --- a/sqlite3_test.go +++ b/sqlite3_test.go @@ -888,6 +888,111 @@ func TestTransaction(t *testing.T) { } } +func TestSerialize(t *testing.T) { + d := SQLiteDriver{} + + srcConn, err := d.Open(":memory:") + if err != nil { + t.Fatal("failed to get database connection:", err) + } + defer srcConn.Close() + sqlite3conn := srcConn.(*SQLiteConn) + + _, err = sqlite3conn.Exec(`CREATE TABLE foo (name string)`, nil) + if err != nil { + t.Fatal("failed to create table:", err) + } + _, err = sqlite3conn.Exec(`INSERT INTO foo(name) VALUES("alice")`, nil) + if err != nil { + t.Fatal("failed to insert record:", err) + } + + // Serialize the database to a file + tempFilename := TempFilename(t) + defer os.Remove(tempFilename) + if err := ioutil.WriteFile(tempFilename, sqlite3conn.Serialize(""), 0644); err != nil { + t.Fatalf("failed to write serialized database to disk") + } + + // Open the SQLite3 file, and test that contents are as expected. + db, err := sql.Open("sqlite3", tempFilename) + if err != nil { + t.Fatal("failed to open database:", err) + } + defer db.Close() + + rows, err := db.Query(`SELECT * FROM foo`) + if err != nil { + t.Fatal("failed to query database:", err) + } + defer rows.Close() + + rows.Next() + + var name string + rows.Scan(&name) + if exp, got := name, "alice"; exp != got { + t.Errorf("Expected %s for fetched result, but got %s:", exp, got) + } +} + +func TestDeserialize(t *testing.T) { + var sqlite3conn *SQLiteConn + d := SQLiteDriver{} + tempFilename := TempFilename(t) + defer os.Remove(tempFilename) + + // Create source database on disk. + conn, err := d.Open(tempFilename) + if err != nil { + t.Fatal("failed to open on-disk database:", err) + } + defer conn.Close() + sqlite3conn = conn.(*SQLiteConn) + _, err = sqlite3conn.Exec(`CREATE TABLE foo (name string)`, nil) + if err != nil { + t.Fatal("failed to create table:", err) + } + _, err = sqlite3conn.Exec(`INSERT INTO foo(name) VALUES("alice")`, nil) + if err != nil { + t.Fatal("failed to insert record:", err) + } + conn.Close() + + // Read database file bytes from disk. + b, err := ioutil.ReadFile(tempFilename) + if err != nil { + t.Fatal("failed to read database file on disk", err) + } + + // Deserialize file contents into memory. + conn, err = d.Open(":memory:") + if err != nil { + t.Fatal("failed to open in-memory database:", err) + } + sqlite3conn = conn.(*SQLiteConn) + defer conn.Close() + if err := sqlite3conn.Deserialize(b, ""); err != nil { + t.Fatal("failed to deserialize database", err) + } + + // Check database contents are as expected. + rows, err := sqlite3conn.Query(`SELECT * FROM foo`, nil) + if err != nil { + t.Fatal("failed to query database:", err) + } + if len(rows.Columns()) != 1 { + t.Fatal("incorrect number of columns returned:", len(rows.Columns())) + } + values := make([]driver.Value, 1) + rows.Next(values) + if v, ok := values[0].(string); !ok { + t.Fatalf("wrong type for value: %T", v) + } else if v != "alice" { + t.Fatal("wrong value returned", v) + } +} + func TestWAL(t *testing.T) { tempFilename := TempFilename(t) defer os.Remove(tempFilename) From 613b8795740dd695d7330dca7613f55c7d66264e Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Sun, 11 Sep 2022 11:56:20 -0400 Subject: [PATCH 02/35] Add support for sqlite_omit_deserialize --- README.md | 1 + sqlite3.go | 44 -------------- sqlite3_opt_serialize.go | 61 +++++++++++++++++++ sqlite3_opt_serialize_omit.go | 11 ++++ sqlite3_opt_serialize_test.go | 110 ++++++++++++++++++++++++++++++++++ 5 files changed, 183 insertions(+), 44 deletions(-) create mode 100644 sqlite3_opt_serialize.go create mode 100644 sqlite3_opt_serialize_omit.go create mode 100644 sqlite3_opt_serialize_test.go diff --git a/README.md b/README.md index 746621f9..63986d18 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,7 @@ go build --tags "icu json1 fts5 secure_delete" | Allow URI Authority | sqlite_allow_uri_authority | URI filenames normally throws an error if the authority section is not either empty or "localhost".

However, if SQLite is compiled with the SQLITE_ALLOW_URI_AUTHORITY compile-time option, then the URI is converted into a Uniform Naming Convention (UNC) filename and passed down to the underlying operating system that way | | App Armor | sqlite_app_armor | When defined, this C-preprocessor macro activates extra code that attempts to detect misuse of the SQLite API, such as passing in NULL pointers to required parameters or using objects after they have been destroyed.

App Armor is not available under `Windows`. | | Disable Load Extensions | sqlite_omit_load_extension | Loading of external extensions is enabled by default.

To disable extension loading add the build tag `sqlite_omit_load_extension`. | +| Disable Serialization | sqlite_omit_deserialize | Serialization and deserialization of a SQLite database is available by default.

To disable it add the build tag `sqlite_omit_deserialize`. | | Foreign Keys | sqlite_foreign_keys | This macro determines whether enforcement of foreign key constraints is enabled or disabled by default for new database connections.

Each database connection can always turn enforcement of foreign key constraints on and off and run-time using the foreign_keys pragma.

Enforcement of foreign key constraints is normally off by default, but if this compile-time parameter is set to 1, enforcement of foreign key constraints will be on by default | | Full Auto Vacuum | sqlite_vacuum_full | Set the default auto vacuum to full | | Incremental Auto Vacuum | sqlite_vacuum_incr | Set the default auto vacuum to incremental | diff --git a/sqlite3.go b/sqlite3.go index cef691e8..5ac95709 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -905,50 +905,6 @@ func (c *SQLiteConn) begin(ctx context.Context) (driver.Tx, error) { return &SQLiteTx{c}, nil } -// Serialize returns a byte slice that is a serialization of the database. -// If the database fails to serialize, a nil slice will be returned. -// -// See https://www.sqlite.org/c3ref/serialize.html -func (c *SQLiteConn) Serialize(schema string) []byte { - if schema == "" { - schema = "main" - } - var zSchema *C.char - zSchema = C.CString(schema) - defer C.free(unsafe.Pointer(zSchema)) - - var sz C.sqlite3_int64 - ptr := C.sqlite3_serialize(c.db, zSchema, &sz, 0) - if ptr == nil { - return nil - } - defer C.sqlite3_free(unsafe.Pointer(ptr)) - return C.GoBytes(unsafe.Pointer(ptr), C.int(sz)) -} - -// Deserialize causes the connection to disconnect from the current database -// and then re-open as an in-memory database based on the contents of the -// byte slice. If deserelization fails, error will contain the return code -// of the underlying SQLite API call. -// -// See https://www.sqlite.org/c3ref/deserialize.html -func (c *SQLiteConn) Deserialize(b []byte, schema string) error { - if schema == "" { - schema = "main" - } - var zSchema *C.char - zSchema = C.CString(schema) - defer C.free(unsafe.Pointer(zSchema)) - - rc := C.sqlite3_deserialize(c.db, zSchema, - (*C.uint8_t)(unsafe.Pointer(&b[0])), - C.sqlite3_int64(len(b)), C.sqlite3_int64(len(b)), 0) - if rc != 0 { - return fmt.Errorf("deserialize failed with return %v", rc) - } - return nil -} - // Open database and return a new connection. // // A pragma can take either zero or one argument. diff --git a/sqlite3_opt_serialize.go b/sqlite3_opt_serialize.go new file mode 100644 index 00000000..979f309e --- /dev/null +++ b/sqlite3_opt_serialize.go @@ -0,0 +1,61 @@ +// +build !sqlite_omit_deserialize + +package sqlite3 + +/* +#ifndef USE_LIBSQLITE3 +#include +#else +#include +#endif +*/ +import "C" + +// Serialize returns a byte slice that is a serialization of the database. +// If the database fails to serialize, a nil slice will be returned. +// +// See https://www.sqlite.org/c3ref/serialize.html +func (c *SQLiteConn) Serialize(schema string) []byte { + if schema == "" { + schema = "main" + } + var zSchema *C.char + zSchema = C.CString(schema) + defer C.free(unsafe.Pointer(zSchema)) + + var sz C.sqlite3_int64 + ptr := C.sqlite3_serialize(c.db, zSchema, &sz, 0) + if ptr == nil { + return nil + } + defer C.sqlite3_free(unsafe.Pointer(ptr)) + return C.GoBytes(unsafe.Pointer(ptr), C.int(sz)) +} + +// Deserialize causes the connection to disconnect from the current database +// and then re-open as an in-memory database based on the contents of the +// byte slice. If deserelization fails, error will contain the return code +// of the underlying SQLite API call. +// +// When this function returns, the connection is referencing database +// data in Go space, so the connection and associated database must be copied +// immediately if it is to be used further. SQLiteConn.Backup() can be used +// to perform this copy. +// +// See https://www.sqlite.org/c3ref/deserialize.html +func (c *SQLiteConn) Deserialize(b []byte, schema string) error { + if schema == "" { + schema = "main" + } + var zSchema *C.char + zSchema = C.CString(schema) + defer C.free(unsafe.Pointer(zSchema)) + + rc := C.sqlite3_deserialize(c.db, zSchema, + (*C.uint8_t)(unsafe.Pointer(&b[0])), + C.sqlite3_int64(len(b)), C.sqlite3_int64(len(b)), 0) + if rc != 0 { + return fmt.Errorf("deserialize failed with return %v", rc) + } + return nil +} diff --git a/sqlite3_opt_serialize_omit.go b/sqlite3_opt_serialize_omit.go new file mode 100644 index 00000000..869186dd --- /dev/null +++ b/sqlite3_opt_serialize_omit.go @@ -0,0 +1,11 @@ +// +build sqlite_omit_deserialize + +package sqlite3 + +func (c *SQLiteConn) Serialize(schema string) []byte { + return nil +} + +func (c *SQLiteConn) Deserialize(b []byte, schema string) error { + return fmt.Errorf("deserialize function not available") +} diff --git a/sqlite3_opt_serialize_test.go b/sqlite3_opt_serialize_test.go new file mode 100644 index 00000000..b3a1d1a5 --- /dev/null +++ b/sqlite3_opt_serialize_test.go @@ -0,0 +1,110 @@ +// +build !sqlite_omit_deserialize + +package sqlite3 + +import "testing" + +func TestSerialize(t *testing.T) { + d := SQLiteDriver{} + + srcConn, err := d.Open(":memory:") + if err != nil { + t.Fatal("failed to get database connection:", err) + } + defer srcConn.Close() + sqlite3conn := srcConn.(*SQLiteConn) + + _, err = sqlite3conn.Exec(`CREATE TABLE foo (name string)`, nil) + if err != nil { + t.Fatal("failed to create table:", err) + } + _, err = sqlite3conn.Exec(`INSERT INTO foo(name) VALUES("alice")`, nil) + if err != nil { + t.Fatal("failed to insert record:", err) + } + + // Serialize the database to a file + tempFilename := TempFilename(t) + defer os.Remove(tempFilename) + if err := ioutil.WriteFile(tempFilename, sqlite3conn.Serialize(""), 0644); err != nil { + t.Fatalf("failed to write serialized database to disk") + } + + // Open the SQLite3 file, and test that contents are as expected. + db, err := sql.Open("sqlite3", tempFilename) + if err != nil { + t.Fatal("failed to open database:", err) + } + defer db.Close() + + rows, err := db.Query(`SELECT * FROM foo`) + if err != nil { + t.Fatal("failed to query database:", err) + } + defer rows.Close() + + rows.Next() + + var name string + rows.Scan(&name) + if exp, got := name, "alice"; exp != got { + t.Errorf("Expected %s for fetched result, but got %s:", exp, got) + } +} + +func TestDeserialize(t *testing.T) { + var sqlite3conn *SQLiteConn + d := SQLiteDriver{} + tempFilename := TempFilename(t) + defer os.Remove(tempFilename) + + // Create source database on disk. + conn, err := d.Open(tempFilename) + if err != nil { + t.Fatal("failed to open on-disk database:", err) + } + defer conn.Close() + sqlite3conn = conn.(*SQLiteConn) + _, err = sqlite3conn.Exec(`CREATE TABLE foo (name string)`, nil) + if err != nil { + t.Fatal("failed to create table:", err) + } + _, err = sqlite3conn.Exec(`INSERT INTO foo(name) VALUES("alice")`, nil) + if err != nil { + t.Fatal("failed to insert record:", err) + } + conn.Close() + + // Read database file bytes from disk. + b, err := ioutil.ReadFile(tempFilename) + if err != nil { + t.Fatal("failed to read database file on disk", err) + } + + // Deserialize file contents into memory. + conn, err = d.Open(":memory:") + if err != nil { + t.Fatal("failed to open in-memory database:", err) + } + sqlite3conn = conn.(*SQLiteConn) + defer conn.Close() + if err := sqlite3conn.Deserialize(b, ""); err != nil { + t.Fatal("failed to deserialize database", err) + } + + // Check database contents are as expected. + rows, err := sqlite3conn.Query(`SELECT * FROM foo`, nil) + if err != nil { + t.Fatal("failed to query database:", err) + } + if len(rows.Columns()) != 1 { + t.Fatal("incorrect number of columns returned:", len(rows.Columns())) + } + values := make([]driver.Value, 1) + rows.Next(values) + if v, ok := values[0].(string); !ok { + t.Fatalf("wrong type for value: %T", v) + } else if v != "alice" { + t.Fatal("wrong value returned", v) + } +} From 706959174e1af95a685002947bd8819eb671a441 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Sun, 11 Sep 2022 12:19:42 -0400 Subject: [PATCH 03/35] Fix up imports --- sqlite3_opt_serialize.go | 7 +++ sqlite3_opt_serialize_test.go | 11 +++- sqlite3_test.go | 105 ---------------------------------- 3 files changed, 15 insertions(+), 108 deletions(-) diff --git a/sqlite3_opt_serialize.go b/sqlite3_opt_serialize.go index 979f309e..9f1e177b 100644 --- a/sqlite3_opt_serialize.go +++ b/sqlite3_opt_serialize.go @@ -8,9 +8,16 @@ package sqlite3 #else #include #endif +#include +#include */ import "C" +import ( + "fmt" + "unsafe" +) + // Serialize returns a byte slice that is a serialization of the database. // If the database fails to serialize, a nil slice will be returned. // diff --git a/sqlite3_opt_serialize_test.go b/sqlite3_opt_serialize_test.go index b3a1d1a5..7e42819e 100644 --- a/sqlite3_opt_serialize_test.go +++ b/sqlite3_opt_serialize_test.go @@ -2,7 +2,12 @@ package sqlite3 -import "testing" +import ( + "database/sql" + "database/sql/driver" + "os" + "testing" +) func TestSerialize(t *testing.T) { d := SQLiteDriver{} @@ -26,7 +31,7 @@ func TestSerialize(t *testing.T) { // Serialize the database to a file tempFilename := TempFilename(t) defer os.Remove(tempFilename) - if err := ioutil.WriteFile(tempFilename, sqlite3conn.Serialize(""), 0644); err != nil { + if err := os.WriteFile(tempFilename, sqlite3conn.Serialize(""), 0644); err != nil { t.Fatalf("failed to write serialized database to disk") } @@ -76,7 +81,7 @@ func TestDeserialize(t *testing.T) { conn.Close() // Read database file bytes from disk. - b, err := ioutil.ReadFile(tempFilename) + b, err := os.ReadFile(tempFilename) if err != nil { t.Fatal("failed to read database file on disk", err) } diff --git a/sqlite3_test.go b/sqlite3_test.go index e5660c58..878ec495 100644 --- a/sqlite3_test.go +++ b/sqlite3_test.go @@ -888,111 +888,6 @@ func TestTransaction(t *testing.T) { } } -func TestSerialize(t *testing.T) { - d := SQLiteDriver{} - - srcConn, err := d.Open(":memory:") - if err != nil { - t.Fatal("failed to get database connection:", err) - } - defer srcConn.Close() - sqlite3conn := srcConn.(*SQLiteConn) - - _, err = sqlite3conn.Exec(`CREATE TABLE foo (name string)`, nil) - if err != nil { - t.Fatal("failed to create table:", err) - } - _, err = sqlite3conn.Exec(`INSERT INTO foo(name) VALUES("alice")`, nil) - if err != nil { - t.Fatal("failed to insert record:", err) - } - - // Serialize the database to a file - tempFilename := TempFilename(t) - defer os.Remove(tempFilename) - if err := ioutil.WriteFile(tempFilename, sqlite3conn.Serialize(""), 0644); err != nil { - t.Fatalf("failed to write serialized database to disk") - } - - // Open the SQLite3 file, and test that contents are as expected. - db, err := sql.Open("sqlite3", tempFilename) - if err != nil { - t.Fatal("failed to open database:", err) - } - defer db.Close() - - rows, err := db.Query(`SELECT * FROM foo`) - if err != nil { - t.Fatal("failed to query database:", err) - } - defer rows.Close() - - rows.Next() - - var name string - rows.Scan(&name) - if exp, got := name, "alice"; exp != got { - t.Errorf("Expected %s for fetched result, but got %s:", exp, got) - } -} - -func TestDeserialize(t *testing.T) { - var sqlite3conn *SQLiteConn - d := SQLiteDriver{} - tempFilename := TempFilename(t) - defer os.Remove(tempFilename) - - // Create source database on disk. - conn, err := d.Open(tempFilename) - if err != nil { - t.Fatal("failed to open on-disk database:", err) - } - defer conn.Close() - sqlite3conn = conn.(*SQLiteConn) - _, err = sqlite3conn.Exec(`CREATE TABLE foo (name string)`, nil) - if err != nil { - t.Fatal("failed to create table:", err) - } - _, err = sqlite3conn.Exec(`INSERT INTO foo(name) VALUES("alice")`, nil) - if err != nil { - t.Fatal("failed to insert record:", err) - } - conn.Close() - - // Read database file bytes from disk. - b, err := ioutil.ReadFile(tempFilename) - if err != nil { - t.Fatal("failed to read database file on disk", err) - } - - // Deserialize file contents into memory. - conn, err = d.Open(":memory:") - if err != nil { - t.Fatal("failed to open in-memory database:", err) - } - sqlite3conn = conn.(*SQLiteConn) - defer conn.Close() - if err := sqlite3conn.Deserialize(b, ""); err != nil { - t.Fatal("failed to deserialize database", err) - } - - // Check database contents are as expected. - rows, err := sqlite3conn.Query(`SELECT * FROM foo`, nil) - if err != nil { - t.Fatal("failed to query database:", err) - } - if len(rows.Columns()) != 1 { - t.Fatal("incorrect number of columns returned:", len(rows.Columns())) - } - values := make([]driver.Value, 1) - rows.Next(values) - if v, ok := values[0].(string); !ok { - t.Fatalf("wrong type for value: %T", v) - } else if v != "alice" { - t.Fatal("wrong value returned", v) - } -} - func TestWAL(t *testing.T) { tempFilename := TempFilename(t) defer os.Remove(tempFilename) From 0cc635c7fc2081f953739cfcad8d2410efa277ba Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Sun, 11 Sep 2022 12:35:33 -0400 Subject: [PATCH 04/35] Add missing import --- sqlite3_opt_serialize_omit.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sqlite3_opt_serialize_omit.go b/sqlite3_opt_serialize_omit.go index 869186dd..7db9f0aa 100644 --- a/sqlite3_opt_serialize_omit.go +++ b/sqlite3_opt_serialize_omit.go @@ -2,6 +2,15 @@ package sqlite3 +/* +#cgo CFLAGS: -DSQLITE_OMIT_DESERIALIZE +*/ +import "C" + +import ( + "fmt" +) + func (c *SQLiteConn) Serialize(schema string) []byte { return nil } From 17653b68c5747eb7f887fb675bbb0b4df4b79a82 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Sun, 11 Sep 2022 12:35:53 -0400 Subject: [PATCH 05/35] go fmt fixes --- sqlite3_opt_serialize_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sqlite3_opt_serialize_test.go b/sqlite3_opt_serialize_test.go index 7e42819e..814c44e2 100644 --- a/sqlite3_opt_serialize_test.go +++ b/sqlite3_opt_serialize_test.go @@ -1,10 +1,11 @@ +//go:build !sqlite_omit_deserialize // +build !sqlite_omit_deserialize package sqlite3 import ( "database/sql" - "database/sql/driver" + "database/sql/driver" "os" "testing" ) From b3777b70f00652fd57aaaaadf7c2a3e9c3b19dd5 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Sun, 11 Sep 2022 12:37:41 -0400 Subject: [PATCH 06/35] Use existing build tag style --- sqlite3_opt_serialize_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sqlite3_opt_serialize_test.go b/sqlite3_opt_serialize_test.go index 814c44e2..58444be4 100644 --- a/sqlite3_opt_serialize_test.go +++ b/sqlite3_opt_serialize_test.go @@ -1,4 +1,3 @@ -//go:build !sqlite_omit_deserialize // +build !sqlite_omit_deserialize package sqlite3 From d4d221279eb683f6499b80c8ca07b3afaff8564d Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Sun, 11 Sep 2022 12:45:18 -0400 Subject: [PATCH 07/35] No Serialize with libsqlite tests --- .github/workflows/go.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml index ec8b30ab..020f9d36 100644 --- a/.github/workflows/go.yaml +++ b/.github/workflows/go.yaml @@ -93,7 +93,7 @@ jobs: shell: msys2 {0} - name: 'Tags: libsqlite3' - run: go build -race -v -tags "libsqlite3" + run: go build -race -v -tags "libsqlite3 sqlite_omit_deserialize" shell: msys2 {0} - name: 'Tags: full' From d607e75d7905deb6068342c12bcf9fca41704862 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Sun, 11 Sep 2022 12:48:28 -0400 Subject: [PATCH 08/35] More libsqlite3 test fixups --- .github/workflows/go.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml index 020f9d36..85b3d68c 100644 --- a/.github/workflows/go.yaml +++ b/.github/workflows/go.yaml @@ -41,7 +41,7 @@ jobs: run: go-acc . -- -race -v -tags "" - name: 'Tags: libsqlite3' - run: go-acc . -- -race -v -tags "libsqlite3" + run: go-acc . -- -race -v -tags "libsqlite3 sqlite_omit_deserialize" - name: 'Tags: full' run: go-acc . -- -race -v -tags "sqlite_allow_uri_authority sqlite_app_armor sqlite_foreign_keys sqlite_fts5 sqlite_icu sqlite_introspect sqlite_json sqlite_os_trace sqlite_preupdate_hook sqlite_secure_delete sqlite_see sqlite_stat4 sqlite_trace sqlite_userauth sqlite_vacuum_incr sqlite_vtable sqlite_unlock_notify sqlite_column_metadata" From 596960adcdf8ae877ebd459755fa309993a4d34c Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 13 Sep 2022 08:37:55 -0400 Subject: [PATCH 09/35] Update expected test output Broken in https://github.com/mattn/go-sqlite3/pull/1085 --- .github/workflows/docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index f2393e6d..83faeb60 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -19,4 +19,4 @@ jobs: run: | cd ./_example/simple docker build -t simple . - docker run simple | grep 99\ こんにちわ世界099 + docker run simple | grep 99\ こんにちは世界099 From 21b8dd91209912eb6ce5f7def6f6da636f6d8eb5 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 13 Sep 2022 09:00:00 -0400 Subject: [PATCH 10/35] Fix up build tags --- sqlite3_opt_serialize.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite3_opt_serialize.go b/sqlite3_opt_serialize.go index 9f1e177b..f456b36f 100644 --- a/sqlite3_opt_serialize.go +++ b/sqlite3_opt_serialize.go @@ -1,4 +1,4 @@ -// +build !sqlite_omit_deserialize +// +build libsqlite3 !sqlite_omit_deserialize package sqlite3 From 03ba6ac436bdaa3be76d230af303203beecaee8d Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 13 Sep 2022 09:18:26 -0400 Subject: [PATCH 11/35] Serialize() should return an explicit error --- sqlite3_opt_serialize.go | 9 ++++----- sqlite3_opt_serialize_omit.go | 4 ++-- sqlite3_opt_serialize_test.go | 6 +++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/sqlite3_opt_serialize.go b/sqlite3_opt_serialize.go index f456b36f..bb759789 100644 --- a/sqlite3_opt_serialize.go +++ b/sqlite3_opt_serialize.go @@ -1,4 +1,4 @@ -// +build libsqlite3 !sqlite_omit_deserialize +// +build !libsqlite3 !sqlite_omit_deserialize package sqlite3 @@ -19,10 +19,9 @@ import ( ) // Serialize returns a byte slice that is a serialization of the database. -// If the database fails to serialize, a nil slice will be returned. // // See https://www.sqlite.org/c3ref/serialize.html -func (c *SQLiteConn) Serialize(schema string) []byte { +func (c *SQLiteConn) Serialize(schema string) ([]byte, error) { if schema == "" { schema = "main" } @@ -33,10 +32,10 @@ func (c *SQLiteConn) Serialize(schema string) []byte { var sz C.sqlite3_int64 ptr := C.sqlite3_serialize(c.db, zSchema, &sz, 0) if ptr == nil { - return nil + return nil, fmt.Errorf("serialize failed") } defer C.sqlite3_free(unsafe.Pointer(ptr)) - return C.GoBytes(unsafe.Pointer(ptr), C.int(sz)) + return C.GoBytes(unsafe.Pointer(ptr), C.int(sz)), nil } // Deserialize causes the connection to disconnect from the current database diff --git a/sqlite3_opt_serialize_omit.go b/sqlite3_opt_serialize_omit.go index 7db9f0aa..b271bf7d 100644 --- a/sqlite3_opt_serialize_omit.go +++ b/sqlite3_opt_serialize_omit.go @@ -11,8 +11,8 @@ import ( "fmt" ) -func (c *SQLiteConn) Serialize(schema string) []byte { - return nil +func (c *SQLiteConn) Serialize(schema string) ([]byte, error) { + return nil, fmt.Errorf("serialize function not available") } func (c *SQLiteConn) Deserialize(b []byte, schema string) error { diff --git a/sqlite3_opt_serialize_test.go b/sqlite3_opt_serialize_test.go index 58444be4..066bd75f 100644 --- a/sqlite3_opt_serialize_test.go +++ b/sqlite3_opt_serialize_test.go @@ -31,7 +31,11 @@ func TestSerialize(t *testing.T) { // Serialize the database to a file tempFilename := TempFilename(t) defer os.Remove(tempFilename) - if err := os.WriteFile(tempFilename, sqlite3conn.Serialize(""), 0644); err != nil { + b, err := sqlite3conn.Serialize("") + if err != nil { + t.Fatalf("failed to serialize database: %s", err) + } + if err := os.WriteFile(tempFilename, b, 0644); err != nil { t.Fatalf("failed to write serialized database to disk") } From 464b7f3bc48db079c140e05ba3f184aad970df95 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 13 Sep 2022 09:28:04 -0400 Subject: [PATCH 12/35] Add missing rows.Close() --- sqlite3_opt_serialize_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sqlite3_opt_serialize_test.go b/sqlite3_opt_serialize_test.go index 066bd75f..18d8e651 100644 --- a/sqlite3_opt_serialize_test.go +++ b/sqlite3_opt_serialize_test.go @@ -106,6 +106,7 @@ func TestDeserialize(t *testing.T) { if err != nil { t.Fatal("failed to query database:", err) } + defer rows.Close() if len(rows.Columns()) != 1 { t.Fatal("incorrect number of columns returned:", len(rows.Columns())) } From 7035040bea862da1f795add188294fbc543142d6 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 13 Sep 2022 09:30:46 -0400 Subject: [PATCH 13/35] More build tag fixes --- sqlite3_opt_serialize_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite3_opt_serialize_test.go b/sqlite3_opt_serialize_test.go index 18d8e651..f79ac2c8 100644 --- a/sqlite3_opt_serialize_test.go +++ b/sqlite3_opt_serialize_test.go @@ -1,4 +1,4 @@ -// +build !sqlite_omit_deserialize +// +build !libsqlite3 !sqlite_omit_deserialize package sqlite3 From ba8ffa0f8780777a4695311300bf11b5b2274960 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 13 Sep 2022 11:18:14 -0400 Subject: [PATCH 14/35] Update build tags --- README.md | 2 +- sqlite3_opt_serialize.go | 2 +- sqlite3_opt_serialize_omit.go | 2 +- sqlite3_opt_serialize_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 058a4b73..23caceb6 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,7 @@ go build --tags "icu json1 fts5 secure_delete" | Allow URI Authority | sqlite_allow_uri_authority | URI filenames normally throws an error if the authority section is not either empty or "localhost".

However, if SQLite is compiled with the SQLITE_ALLOW_URI_AUTHORITY compile-time option, then the URI is converted into a Uniform Naming Convention (UNC) filename and passed down to the underlying operating system that way | | App Armor | sqlite_app_armor | When defined, this C-preprocessor macro activates extra code that attempts to detect misuse of the SQLite API, such as passing in NULL pointers to required parameters or using objects after they have been destroyed.

App Armor is not available under `Windows`. | | Disable Load Extensions | sqlite_omit_load_extension | Loading of external extensions is enabled by default.

To disable extension loading add the build tag `sqlite_omit_load_extension`. | -| Disable Serialization | sqlite_omit_deserialize | Serialization and deserialization of a SQLite database is available by default.

To disable it add the build tag `sqlite_omit_deserialize`. | +| Enable Serialization with `libsqlite3` | sqlite_serialize | Serialization and deserialization of a SQLite database is available by default, unless the build tag `libsqlite3` is set..

To enable this functionality even if `libsqlite3` is set, add the build tag `sqlite_serialize`. | | Foreign Keys | sqlite_foreign_keys | This macro determines whether enforcement of foreign key constraints is enabled or disabled by default for new database connections.

Each database connection can always turn enforcement of foreign key constraints on and off and run-time using the foreign_keys pragma.

Enforcement of foreign key constraints is normally off by default, but if this compile-time parameter is set to 1, enforcement of foreign key constraints will be on by default | | Full Auto Vacuum | sqlite_vacuum_full | Set the default auto vacuum to full | | Incremental Auto Vacuum | sqlite_vacuum_incr | Set the default auto vacuum to incremental | diff --git a/sqlite3_opt_serialize.go b/sqlite3_opt_serialize.go index bb759789..ba705237 100644 --- a/sqlite3_opt_serialize.go +++ b/sqlite3_opt_serialize.go @@ -1,4 +1,4 @@ -// +build !libsqlite3 !sqlite_omit_deserialize +// +build !libsqlite3 sqlite_serialize package sqlite3 diff --git a/sqlite3_opt_serialize_omit.go b/sqlite3_opt_serialize_omit.go index b271bf7d..260fac6b 100644 --- a/sqlite3_opt_serialize_omit.go +++ b/sqlite3_opt_serialize_omit.go @@ -1,4 +1,4 @@ -// +build sqlite_omit_deserialize +// +build libsqlite3 !sqlite_serialize package sqlite3 diff --git a/sqlite3_opt_serialize_test.go b/sqlite3_opt_serialize_test.go index f79ac2c8..652d9b2f 100644 --- a/sqlite3_opt_serialize_test.go +++ b/sqlite3_opt_serialize_test.go @@ -1,4 +1,4 @@ -// +build !libsqlite3 !sqlite_omit_deserialize +// +build !libsqlite3 sqlite_serialize package sqlite3 From 270263f8fbe7ba6a83cd12c79984e2c354ec6764 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 13 Sep 2022 11:18:32 -0400 Subject: [PATCH 15/35] Update README for build tags --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 23caceb6..4d7fc0fc 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,7 @@ go build --tags "icu json1 fts5 secure_delete" | Allow URI Authority | sqlite_allow_uri_authority | URI filenames normally throws an error if the authority section is not either empty or "localhost".

However, if SQLite is compiled with the SQLITE_ALLOW_URI_AUTHORITY compile-time option, then the URI is converted into a Uniform Naming Convention (UNC) filename and passed down to the underlying operating system that way | | App Armor | sqlite_app_armor | When defined, this C-preprocessor macro activates extra code that attempts to detect misuse of the SQLite API, such as passing in NULL pointers to required parameters or using objects after they have been destroyed.

App Armor is not available under `Windows`. | | Disable Load Extensions | sqlite_omit_load_extension | Loading of external extensions is enabled by default.

To disable extension loading add the build tag `sqlite_omit_load_extension`. | -| Enable Serialization with `libsqlite3` | sqlite_serialize | Serialization and deserialization of a SQLite database is available by default, unless the build tag `libsqlite3` is set..

To enable this functionality even if `libsqlite3` is set, add the build tag `sqlite_serialize`. | +| Enable Serialization with `libsqlite3` | sqlite_serialize | Serialization and deserialization of a SQLite database is available by default, unless the build tag `libsqlite3` is set.

To enable this functionality even if `libsqlite3` is set, add the build tag `sqlite_serialize`. | | Foreign Keys | sqlite_foreign_keys | This macro determines whether enforcement of foreign key constraints is enabled or disabled by default for new database connections.

Each database connection can always turn enforcement of foreign key constraints on and off and run-time using the foreign_keys pragma.

Enforcement of foreign key constraints is normally off by default, but if this compile-time parameter is set to 1, enforcement of foreign key constraints will be on by default | | Full Auto Vacuum | sqlite_vacuum_full | Set the default auto vacuum to full | | Incremental Auto Vacuum | sqlite_vacuum_incr | Set the default auto vacuum to incremental | From 491eab1b6527728dc888dc789fa3a203793c1791 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 13 Sep 2022 11:25:37 -0400 Subject: [PATCH 16/35] AND logic for build tags --- sqlite3_opt_serialize.go | 37 ++++++++++++++++++++++++++--------- sqlite3_opt_serialize_omit.go | 3 ++- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/sqlite3_opt_serialize.go b/sqlite3_opt_serialize.go index ba705237..431a9fe5 100644 --- a/sqlite3_opt_serialize.go +++ b/sqlite3_opt_serialize.go @@ -15,6 +15,7 @@ import "C" import ( "fmt" + "reflect" "unsafe" ) @@ -35,7 +36,23 @@ func (c *SQLiteConn) Serialize(schema string) ([]byte, error) { return nil, fmt.Errorf("serialize failed") } defer C.sqlite3_free(unsafe.Pointer(ptr)) - return C.GoBytes(unsafe.Pointer(ptr), C.int(sz)), nil + + if C.sizeof_int < 64 { + maxSize := C.sqlite3_int64(1)< maxSize { + return nil, fmt.Errorf("sqlite3: serialized database is too large (%d bytes)", maxSize) + } + } + + cBuf := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ + Data: uintptr(unsafe.Pointer(ptr)), + Len: int(sz), + Cap: int(sz), + })) + + res := make([]byte, int(sz)) + copy(res, cBuf) + return res, nil } // Deserialize causes the connection to disconnect from the current database @@ -43,11 +60,6 @@ func (c *SQLiteConn) Serialize(schema string) ([]byte, error) { // byte slice. If deserelization fails, error will contain the return code // of the underlying SQLite API call. // -// When this function returns, the connection is referencing database -// data in Go space, so the connection and associated database must be copied -// immediately if it is to be used further. SQLiteConn.Backup() can be used -// to perform this copy. -// // See https://www.sqlite.org/c3ref/deserialize.html func (c *SQLiteConn) Deserialize(b []byte, schema string) error { if schema == "" { @@ -57,11 +69,18 @@ func (c *SQLiteConn) Deserialize(b []byte, schema string) error { zSchema = C.CString(schema) defer C.free(unsafe.Pointer(zSchema)) - rc := C.sqlite3_deserialize(c.db, zSchema, - (*C.uint8_t)(unsafe.Pointer(&b[0])), - C.sqlite3_int64(len(b)), C.sqlite3_int64(len(b)), 0) + tmpBuf := (*C.uchar)(C.sqlite3_malloc64(C.sqlite3_uint64(len(b)))) + cBuf := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ + Data: uintptr(unsafe.Pointer(tmpBuf)), + Len: len(b), + Cap: len(b), + })) + copy(cBuf, b) + + rc := C.sqlite3_deserialize(c.db, zSchema, tmpBuf, C.sqlite3_int64(len(b)), 0, 0) if rc != 0 { return fmt.Errorf("deserialize failed with return %v", rc) } + return nil } diff --git a/sqlite3_opt_serialize_omit.go b/sqlite3_opt_serialize_omit.go index 260fac6b..d02ecd1a 100644 --- a/sqlite3_opt_serialize_omit.go +++ b/sqlite3_opt_serialize_omit.go @@ -1,4 +1,5 @@ -// +build libsqlite3 !sqlite_serialize +// +build libsqlite3 +// +build !sqlite_serialize package sqlite3 From ac1d03c5fa6cdaebe12a233696a6554765c3fd8d Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 13 Sep 2022 11:26:15 -0400 Subject: [PATCH 17/35] Remove blank line --- sqlite3_opt_serialize.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sqlite3_opt_serialize.go b/sqlite3_opt_serialize.go index 431a9fe5..e1436042 100644 --- a/sqlite3_opt_serialize.go +++ b/sqlite3_opt_serialize.go @@ -81,6 +81,5 @@ func (c *SQLiteConn) Deserialize(b []byte, schema string) error { if rc != 0 { return fmt.Errorf("deserialize failed with return %v", rc) } - return nil } From c6d62962d64e4346254fbdac2c84ffdca41d2ff7 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 13 Sep 2022 15:44:39 -0400 Subject: [PATCH 18/35] More memory management fixes --- sqlite3_opt_serialize.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/sqlite3_opt_serialize.go b/sqlite3_opt_serialize.go index e1436042..86cd0ceb 100644 --- a/sqlite3_opt_serialize.go +++ b/sqlite3_opt_serialize.go @@ -15,10 +15,15 @@ import "C" import ( "fmt" + "math" "reflect" "unsafe" ) +const ( + SQLITEDeserializeFreeOnClose = 1 +) + // Serialize returns a byte slice that is a serialization of the database. // // See https://www.sqlite.org/c3ref/serialize.html @@ -37,11 +42,8 @@ func (c *SQLiteConn) Serialize(schema string) ([]byte, error) { } defer C.sqlite3_free(unsafe.Pointer(ptr)) - if C.sizeof_int < 64 { - maxSize := C.sqlite3_int64(1)< maxSize { - return nil, fmt.Errorf("sqlite3: serialized database is too large (%d bytes)", maxSize) - } + if sz > C.sqlite3_int64(math.MaxInt) { + return nil, fmt.Errorf("serialized database is too large (%d bytes)", sz) } cBuf := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ @@ -77,7 +79,8 @@ func (c *SQLiteConn) Deserialize(b []byte, schema string) error { })) copy(cBuf, b) - rc := C.sqlite3_deserialize(c.db, zSchema, tmpBuf, C.sqlite3_int64(len(b)), 0, 0) + rc := C.sqlite3_deserialize(c.db, zSchema, tmpBuf, C.sqlite3_int64(len(b)), + C.sqlite3_int64(len(b)), SQLITEDeserializeFreeOnClose) if rc != 0 { return fmt.Errorf("deserialize failed with return %v", rc) } From 99fe433a85e7ff71009fa67a2968869c15db7fdb Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 13 Sep 2022 22:02:13 -0400 Subject: [PATCH 19/35] Update sqlite3_opt_serialize.go Co-authored-by: rittneje --- sqlite3_opt_serialize.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite3_opt_serialize.go b/sqlite3_opt_serialize.go index 86cd0ceb..74b0ea7e 100644 --- a/sqlite3_opt_serialize.go +++ b/sqlite3_opt_serialize.go @@ -59,7 +59,7 @@ func (c *SQLiteConn) Serialize(schema string) ([]byte, error) { // Deserialize causes the connection to disconnect from the current database // and then re-open as an in-memory database based on the contents of the -// byte slice. If deserelization fails, error will contain the return code +// byte slice. If deserialization fails, error will contain the return code // of the underlying SQLite API call. // // See https://www.sqlite.org/c3ref/deserialize.html From 7ed94a945a0db7b2cfdc970eca264980b9495aef Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 13 Sep 2022 22:02:29 -0400 Subject: [PATCH 20/35] Update sqlite3_opt_serialize_omit.go Co-authored-by: rittneje --- sqlite3_opt_serialize_omit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite3_opt_serialize_omit.go b/sqlite3_opt_serialize_omit.go index d02ecd1a..ecc593ac 100644 --- a/sqlite3_opt_serialize_omit.go +++ b/sqlite3_opt_serialize_omit.go @@ -13,7 +13,7 @@ import ( ) func (c *SQLiteConn) Serialize(schema string) ([]byte, error) { - return nil, fmt.Errorf("serialize function not available") + return nil, errors.New("sqlite3: Serialize requires the sqlite_serialize build tag when using the libsqlite3 build tag") } func (c *SQLiteConn) Deserialize(b []byte, schema string) error { From 03bb5743940db792ccb20af63aed279b18f7f209 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 13 Sep 2022 22:03:48 -0400 Subject: [PATCH 21/35] More fixes --- .github/workflows/go.yaml | 4 ++-- sqlite3_opt_serialize.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml index 85b3d68c..ec8b30ab 100644 --- a/.github/workflows/go.yaml +++ b/.github/workflows/go.yaml @@ -41,7 +41,7 @@ jobs: run: go-acc . -- -race -v -tags "" - name: 'Tags: libsqlite3' - run: go-acc . -- -race -v -tags "libsqlite3 sqlite_omit_deserialize" + run: go-acc . -- -race -v -tags "libsqlite3" - name: 'Tags: full' run: go-acc . -- -race -v -tags "sqlite_allow_uri_authority sqlite_app_armor sqlite_foreign_keys sqlite_fts5 sqlite_icu sqlite_introspect sqlite_json sqlite_os_trace sqlite_preupdate_hook sqlite_secure_delete sqlite_see sqlite_stat4 sqlite_trace sqlite_userauth sqlite_vacuum_incr sqlite_vtable sqlite_unlock_notify sqlite_column_metadata" @@ -93,7 +93,7 @@ jobs: shell: msys2 {0} - name: 'Tags: libsqlite3' - run: go build -race -v -tags "libsqlite3 sqlite_omit_deserialize" + run: go build -race -v -tags "libsqlite3" shell: msys2 {0} - name: 'Tags: full' diff --git a/sqlite3_opt_serialize.go b/sqlite3_opt_serialize.go index 86cd0ceb..9ce334c0 100644 --- a/sqlite3_opt_serialize.go +++ b/sqlite3_opt_serialize.go @@ -80,7 +80,7 @@ func (c *SQLiteConn) Deserialize(b []byte, schema string) error { copy(cBuf, b) rc := C.sqlite3_deserialize(c.db, zSchema, tmpBuf, C.sqlite3_int64(len(b)), - C.sqlite3_int64(len(b)), SQLITEDeserializeFreeOnClose) + C.sqlite3_int64(len(b)), C.SQLITE_DESERIALIZE_FREEONCLOSE) if rc != 0 { return fmt.Errorf("deserialize failed with return %v", rc) } From d2e464d3ee2862755b52f237f3685f59ef70e882 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 13 Sep 2022 22:04:51 -0400 Subject: [PATCH 22/35] Remove unused const --- sqlite3_opt_serialize.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sqlite3_opt_serialize.go b/sqlite3_opt_serialize.go index 0bdcf370..f8100bd9 100644 --- a/sqlite3_opt_serialize.go +++ b/sqlite3_opt_serialize.go @@ -20,10 +20,6 @@ import ( "unsafe" ) -const ( - SQLITEDeserializeFreeOnClose = 1 -) - // Serialize returns a byte slice that is a serialization of the database. // // See https://www.sqlite.org/c3ref/serialize.html From 469d82ca9decdbe9f0283060f668d4eeb30bdf79 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 13 Sep 2022 22:10:50 -0400 Subject: [PATCH 23/35] Update sqlite3_opt_serialize_omit.go Co-authored-by: rittneje --- sqlite3_opt_serialize_omit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite3_opt_serialize_omit.go b/sqlite3_opt_serialize_omit.go index ecc593ac..e448a9fe 100644 --- a/sqlite3_opt_serialize_omit.go +++ b/sqlite3_opt_serialize_omit.go @@ -1,4 +1,4 @@ -// +build libsqlite3 +// +build libsqlite3,!sqlite_serialize // +build !sqlite_serialize package sqlite3 From a4046585a92be39a5eaeb4574c65324fe7fb30da Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 13 Sep 2022 22:12:38 -0400 Subject: [PATCH 24/35] Fix up build tags --- sqlite3_opt_serialize_omit.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sqlite3_opt_serialize_omit.go b/sqlite3_opt_serialize_omit.go index e448a9fe..594e9113 100644 --- a/sqlite3_opt_serialize_omit.go +++ b/sqlite3_opt_serialize_omit.go @@ -1,5 +1,4 @@ // +build libsqlite3,!sqlite_serialize -// +build !sqlite_serialize package sqlite3 From 0740ee999d6f7b20bd55f215da901db49a8fa007 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 13 Sep 2022 22:13:39 -0400 Subject: [PATCH 25/35] Fix error messages --- sqlite3_opt_serialize_omit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite3_opt_serialize_omit.go b/sqlite3_opt_serialize_omit.go index 594e9113..5ccdbf20 100644 --- a/sqlite3_opt_serialize_omit.go +++ b/sqlite3_opt_serialize_omit.go @@ -16,5 +16,5 @@ func (c *SQLiteConn) Serialize(schema string) ([]byte, error) { } func (c *SQLiteConn) Deserialize(b []byte, schema string) error { - return fmt.Errorf("deserialize function not available") + return nil, errors.New("sqlite3: Deserialize requires the sqlite_serialize build tag when using the libsqlite3 build tag") } From 4516d61c343d140cd3ec18ccac1bee9d5fda1b62 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 13 Sep 2022 22:20:17 -0400 Subject: [PATCH 26/35] Remove unused import --- sqlite3_opt_serialize_omit.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sqlite3_opt_serialize_omit.go b/sqlite3_opt_serialize_omit.go index 5ccdbf20..9d394360 100644 --- a/sqlite3_opt_serialize_omit.go +++ b/sqlite3_opt_serialize_omit.go @@ -7,10 +7,6 @@ package sqlite3 */ import "C" -import ( - "fmt" -) - func (c *SQLiteConn) Serialize(schema string) ([]byte, error) { return nil, errors.New("sqlite3: Serialize requires the sqlite_serialize build tag when using the libsqlite3 build tag") } From a0dc1efdeeedf47b324462db48911f430392a169 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 13 Sep 2022 22:22:17 -0400 Subject: [PATCH 27/35] Import "errors" --- sqlite3_opt_serialize_omit.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sqlite3_opt_serialize_omit.go b/sqlite3_opt_serialize_omit.go index 9d394360..45ee600b 100644 --- a/sqlite3_opt_serialize_omit.go +++ b/sqlite3_opt_serialize_omit.go @@ -2,6 +2,10 @@ package sqlite3 +import ( + "errors" +) + /* #cgo CFLAGS: -DSQLITE_OMIT_DESERIALIZE */ From f1d96d70f8a7090619bf960d8e015e94521b02a1 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 13 Sep 2022 22:23:06 -0400 Subject: [PATCH 28/35] go fmt --- sqlite3_opt_serialize_omit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite3_opt_serialize_omit.go b/sqlite3_opt_serialize_omit.go index 45ee600b..40a3d4ca 100644 --- a/sqlite3_opt_serialize_omit.go +++ b/sqlite3_opt_serialize_omit.go @@ -11,7 +11,7 @@ import ( */ import "C" -func (c *SQLiteConn) Serialize(schema string) ([]byte, error) { +func (c *SQLiteConn) Serialize(schema string) ([]byte, error) { return nil, errors.New("sqlite3: Serialize requires the sqlite_serialize build tag when using the libsqlite3 build tag") } From ec6e50d06db7d67a56aa27c03f418d8d0e4d62cb Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 13 Sep 2022 22:27:04 -0400 Subject: [PATCH 29/35] Fix copy 'n' paste error --- sqlite3_opt_serialize_omit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite3_opt_serialize_omit.go b/sqlite3_opt_serialize_omit.go index 40a3d4ca..b154dd34 100644 --- a/sqlite3_opt_serialize_omit.go +++ b/sqlite3_opt_serialize_omit.go @@ -16,5 +16,5 @@ func (c *SQLiteConn) Serialize(schema string) ([]byte, error) { } func (c *SQLiteConn) Deserialize(b []byte, schema string) error { - return nil, errors.New("sqlite3: Deserialize requires the sqlite_serialize build tag when using the libsqlite3 build tag") + return errors.New("sqlite3: Deserialize requires the sqlite_serialize build tag when using the libsqlite3 build tag") } From 3bbd5c18303eef91fbb30d55980cbb3f969ff3d6 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Mon, 19 Sep 2022 08:50:03 -0400 Subject: [PATCH 30/35] Address latest comments --- sqlite3_opt_serialize.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sqlite3_opt_serialize.go b/sqlite3_opt_serialize.go index f8100bd9..7aac18aa 100644 --- a/sqlite3_opt_serialize.go +++ b/sqlite3_opt_serialize.go @@ -55,8 +55,7 @@ func (c *SQLiteConn) Serialize(schema string) ([]byte, error) { // Deserialize causes the connection to disconnect from the current database // and then re-open as an in-memory database based on the contents of the -// byte slice. If deserialization fails, error will contain the return code -// of the underlying SQLite API call. +a// byte slice. // // See https://www.sqlite.org/c3ref/deserialize.html func (c *SQLiteConn) Deserialize(b []byte, schema string) error { @@ -77,7 +76,7 @@ func (c *SQLiteConn) Deserialize(b []byte, schema string) error { rc := C.sqlite3_deserialize(c.db, zSchema, tmpBuf, C.sqlite3_int64(len(b)), C.sqlite3_int64(len(b)), C.SQLITE_DESERIALIZE_FREEONCLOSE) - if rc != 0 { + if rc != C.SQLITE_OK { return fmt.Errorf("deserialize failed with return %v", rc) } return nil From 96afe68d4196cd1a5dbc0a2b153bb0c3e87de479 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Mon, 19 Sep 2022 08:50:59 -0400 Subject: [PATCH 31/35] Correct typo --- sqlite3_opt_serialize.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sqlite3_opt_serialize.go b/sqlite3_opt_serialize.go index 7aac18aa..2560c43a 100644 --- a/sqlite3_opt_serialize.go +++ b/sqlite3_opt_serialize.go @@ -53,9 +53,8 @@ func (c *SQLiteConn) Serialize(schema string) ([]byte, error) { return res, nil } -// Deserialize causes the connection to disconnect from the current database -// and then re-open as an in-memory database based on the contents of the -a// byte slice. +// Deserialize causes the connection to disconnect from the current database and +// then re-open as an in-memory database based on the contents of the byte slice. // // See https://www.sqlite.org/c3ref/deserialize.html func (c *SQLiteConn) Deserialize(b []byte, schema string) error { From 6b691944f171f52a261c15883c833a6b00e89a22 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Fri, 7 Oct 2022 09:16:45 -0400 Subject: [PATCH 32/35] Write Serialization test using database/sql --- sqlite3_opt_serialize_test.go | 142 +++++++++++++++------------------- 1 file changed, 62 insertions(+), 80 deletions(-) diff --git a/sqlite3_opt_serialize_test.go b/sqlite3_opt_serialize_test.go index 652d9b2f..548dd072 100644 --- a/sqlite3_opt_serialize_test.go +++ b/sqlite3_opt_serialize_test.go @@ -1,120 +1,102 @@ +//go:build !libsqlite3 || sqlite_serialize // +build !libsqlite3 sqlite_serialize package sqlite3 import ( "database/sql" - "database/sql/driver" "os" "testing" ) -func TestSerialize(t *testing.T) { - d := SQLiteDriver{} +func TestSerializeDeserialize(t *testing.T) { + // The driver's connection will be needed in order to serialization and deserialization + driverName := "TestSerializeDeserialize" + driverConns := []*SQLiteConn{} + sql.Register(driverName, &SQLiteDriver{ + ConnectHook: func(conn *SQLiteConn) error { + driverConns = append(driverConns, conn) + return nil + }, + }) - srcConn, err := d.Open(":memory:") + // Connect to the source database. + srcTempFilename := TempFilename(t) + defer os.Remove(srcTempFilename) + srcDb, err := sql.Open(driverName, srcTempFilename) if err != nil { - t.Fatal("failed to get database connection:", err) + t.Fatal("Failed to open the source database:", err) } - defer srcConn.Close() - sqlite3conn := srcConn.(*SQLiteConn) - - _, err = sqlite3conn.Exec(`CREATE TABLE foo (name string)`, nil) - if err != nil { - t.Fatal("failed to create table:", err) - } - _, err = sqlite3conn.Exec(`INSERT INTO foo(name) VALUES("alice")`, nil) + defer srcDb.Close() + err = srcDb.Ping() if err != nil { - t.Fatal("failed to insert record:", err) + t.Fatal("Failed to connect to the source database:", err) } - // Serialize the database to a file - tempFilename := TempFilename(t) - defer os.Remove(tempFilename) - b, err := sqlite3conn.Serialize("") + // Connect to the destination database. + destTempFilename := TempFilename(t) + defer os.Remove(destTempFilename) + destDb, err := sql.Open(driverName, destTempFilename) if err != nil { - t.Fatalf("failed to serialize database: %s", err) - } - if err := os.WriteFile(tempFilename, b, 0644); err != nil { - t.Fatalf("failed to write serialized database to disk") + t.Fatal("Failed to open the destination database:", err) } - - // Open the SQLite3 file, and test that contents are as expected. - db, err := sql.Open("sqlite3", tempFilename) + defer destDb.Close() + err = destDb.Ping() if err != nil { - t.Fatal("failed to open database:", err) + t.Fatal("Failed to connect to the destination database:", err) } - defer db.Close() - rows, err := db.Query(`SELECT * FROM foo`) - if err != nil { - t.Fatal("failed to query database:", err) + // Check the driver connections. + if len(driverConns) != 2 { + t.Fatalf("Expected 2 driver connections, but found %v.", len(driverConns)) } - defer rows.Close() - - rows.Next() - - var name string - rows.Scan(&name) - if exp, got := name, "alice"; exp != got { - t.Errorf("Expected %s for fetched result, but got %s:", exp, got) + srcDbDriverConn := driverConns[0] + if srcDbDriverConn == nil { + t.Fatal("The source database driver connection is nil.") } -} - -func TestDeserialize(t *testing.T) { - var sqlite3conn *SQLiteConn - d := SQLiteDriver{} - tempFilename := TempFilename(t) - defer os.Remove(tempFilename) - - // Create source database on disk. - conn, err := d.Open(tempFilename) - if err != nil { - t.Fatal("failed to open on-disk database:", err) + destDbDriverConn := driverConns[1] + if destDbDriverConn == nil { + t.Fatal("The destination database driver connection is nil.") } - defer conn.Close() - sqlite3conn = conn.(*SQLiteConn) - _, err = sqlite3conn.Exec(`CREATE TABLE foo (name string)`, nil) + + // Write data to source database. + _, err = srcDb.Exec(`CREATE TABLE foo (name string)`) if err != nil { - t.Fatal("failed to create table:", err) + t.Fatal("Failed to create table in source database:", err) } - _, err = sqlite3conn.Exec(`INSERT INTO foo(name) VALUES("alice")`, nil) + _, err = srcDb.Exec(`INSERT INTO foo(name) VALUES("alice")`) if err != nil { - t.Fatal("failed to insert record:", err) + t.Fatal("Failed to insert data into source database", err) } - conn.Close() - // Read database file bytes from disk. - b, err := os.ReadFile(tempFilename) + // Serialize source database + b, err := srcDbDriverConn.Serialize("") if err != nil { - t.Fatal("failed to read database file on disk", err) + t.Fatalf("Failed to serialize source database: %s", err) } - // Deserialize file contents into memory. - conn, err = d.Open(":memory:") + // Confirm that the destination database is initially empty. + var destTableCount int + err = destDb.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table'").Scan(&destTableCount) if err != nil { - t.Fatal("failed to open in-memory database:", err) + t.Fatal("Failed to check the destination table count:", err) } - sqlite3conn = conn.(*SQLiteConn) - defer conn.Close() - if err := sqlite3conn.Deserialize(b, ""); err != nil { - t.Fatal("failed to deserialize database", err) + if destTableCount != 0 { + t.Fatalf("The destination database is not empty; %v table(s) found.", destTableCount) } - // Check database contents are as expected. - rows, err := sqlite3conn.Query(`SELECT * FROM foo`, nil) - if err != nil { - t.Fatal("failed to query database:", err) + // Deserialize to destination database + if err := destDbDriverConn.Deserialize(b, ""); err != nil { + t.Fatal("Failed to deserialize to destination database", err) } - defer rows.Close() - if len(rows.Columns()) != 1 { - t.Fatal("incorrect number of columns returned:", len(rows.Columns())) + + // Confirm that destination database has been loaded correctly. + var destRowCount int + err = destDb.QueryRow(`SELECT COUNT(*) FROM foo`).Scan(&destRowCount) + if err != nil { + t.Fatal("Failed to count rows in destination database table", err) } - values := make([]driver.Value, 1) - rows.Next(values) - if v, ok := values[0].(string); !ok { - t.Fatalf("wrong type for value: %T", v) - } else if v != "alice" { - t.Fatal("wrong value returned", v) + if destRowCount != 1 { + t.Fatalf("Destination table does not have the expected records") } } From 887388380fae08ae7b60e77d1bb2f075bf05993a Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Fri, 7 Oct 2022 09:20:48 -0400 Subject: [PATCH 33/35] Use existing style for build tags --- sqlite3_opt_serialize_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sqlite3_opt_serialize_test.go b/sqlite3_opt_serialize_test.go index 548dd072..43b6c7a0 100644 --- a/sqlite3_opt_serialize_test.go +++ b/sqlite3_opt_serialize_test.go @@ -1,4 +1,3 @@ -//go:build !libsqlite3 || sqlite_serialize // +build !libsqlite3 sqlite_serialize package sqlite3 From 2f60ff24dd426d5a2675e0ee1895cac34dd1520f Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 8 Nov 2022 08:46:50 -0500 Subject: [PATCH 34/35] Update test as per PR comments --- sqlite3_opt_serialize_test.go | 51 ++++++++++++++++------------------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/sqlite3_opt_serialize_test.go b/sqlite3_opt_serialize_test.go index 43b6c7a0..145a9027 100644 --- a/sqlite3_opt_serialize_test.go +++ b/sqlite3_opt_serialize_test.go @@ -9,16 +9,6 @@ import ( ) func TestSerializeDeserialize(t *testing.T) { - // The driver's connection will be needed in order to serialization and deserialization - driverName := "TestSerializeDeserialize" - driverConns := []*SQLiteConn{} - sql.Register(driverName, &SQLiteDriver{ - ConnectHook: func(conn *SQLiteConn) error { - driverConns = append(driverConns, conn) - return nil - }, - }) - // Connect to the source database. srcTempFilename := TempFilename(t) defer os.Remove(srcTempFilename) @@ -45,19 +35,6 @@ func TestSerializeDeserialize(t *testing.T) { t.Fatal("Failed to connect to the destination database:", err) } - // Check the driver connections. - if len(driverConns) != 2 { - t.Fatalf("Expected 2 driver connections, but found %v.", len(driverConns)) - } - srcDbDriverConn := driverConns[0] - if srcDbDriverConn == nil { - t.Fatal("The source database driver connection is nil.") - } - destDbDriverConn := driverConns[1] - if destDbDriverConn == nil { - t.Fatal("The destination database driver connection is nil.") - } - // Write data to source database. _, err = srcDb.Exec(`CREATE TABLE foo (name string)`) if err != nil { @@ -68,10 +45,20 @@ func TestSerializeDeserialize(t *testing.T) { t.Fatal("Failed to insert data into source database", err) } - // Serialize source database - b, err := srcDbDriverConn.Serialize("") + // Serialize the source database + srcConn, err := srcDb.Conn(context.Background()) if err != nil { - t.Fatalf("Failed to serialize source database: %s", err) + t.Fatal("Failed to get connection to source database:", err) + } + defer srcConn.Close() + + var serialized []byte + if err := srcConn.Raw(func(raw interface{}) error { + var err error + serialized, err = raw.(*sqlite3.SQLiteConn).Serialize("") + return err + }); err != nil { + t.Fatal(err) } // Confirm that the destination database is initially empty. @@ -85,8 +72,16 @@ func TestSerializeDeserialize(t *testing.T) { } // Deserialize to destination database - if err := destDbDriverConn.Deserialize(b, ""); err != nil { - t.Fatal("Failed to deserialize to destination database", err) + destConn, err := destDb.Conn(context.Background()) + if err != nil { + t.Fatal("Failed to get connection to destination database:", err) + } + defer destConn.Close() + + if err := destConn.Raw(func(raw interface{}) error { + return raw.(*sqlite3.SQLiteConn).Deserialize(serialized, "") + }); err != nil { + t.Fatal(err) } // Confirm that destination database has been loaded correctly. From 9ec6bc6ca0cc02b3952f5847fa1b4697a573ef1a Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Tue, 8 Nov 2022 08:50:53 -0500 Subject: [PATCH 35/35] Fix up Serialize unit test --- sqlite3_opt_serialize_test.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/sqlite3_opt_serialize_test.go b/sqlite3_opt_serialize_test.go index 145a9027..624c5a94 100644 --- a/sqlite3_opt_serialize_test.go +++ b/sqlite3_opt_serialize_test.go @@ -3,6 +3,7 @@ package sqlite3 import ( + "context" "database/sql" "os" "testing" @@ -55,11 +56,12 @@ func TestSerializeDeserialize(t *testing.T) { var serialized []byte if err := srcConn.Raw(func(raw interface{}) error { var err error - serialized, err = raw.(*sqlite3.SQLiteConn).Serialize("") + serialized, err = raw.(*SQLiteConn).Serialize("") return err }); err != nil { - t.Fatal(err) + t.Fatal("Failed to serialize source database:", err) } + srcConn.Close() // Confirm that the destination database is initially empty. var destTableCount int @@ -79,10 +81,11 @@ func TestSerializeDeserialize(t *testing.T) { defer destConn.Close() if err := destConn.Raw(func(raw interface{}) error { - return raw.(*sqlite3.SQLiteConn).Deserialize(serialized, "") + return raw.(*SQLiteConn).Deserialize(serialized, "") }); err != nil { - t.Fatal(err) + t.Fatal("Failed to deserialize source database:", err) } + destConn.Close() // Confirm that destination database has been loaded correctly. var destRowCount int