From 7ac0064e822156a17a6b598957ddf5e0287f8288 Mon Sep 17 00:00:00 2001 From: Julien Schmidt Date: Wed, 18 Jul 2018 16:30:26 +0200 Subject: [PATCH 01/93] try to fix handling of empty auth plugin names (#835) --- driver.go | 3 +++ packets.go | 9 +++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/driver.go b/driver.go index 8c35de73c..ba1297825 100644 --- a/driver.go +++ b/driver.go @@ -105,6 +105,9 @@ func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { mc.cleanup() return nil, err } + if plugin == "" { + plugin = defaultAuthPlugin + } // Send Client Authentication Packet authResp, addNUL, err := mc.auth(authData, plugin) diff --git a/packets.go b/packets.go index f99934e73..170aaa02b 100644 --- a/packets.go +++ b/packets.go @@ -154,15 +154,15 @@ func (mc *mysqlConn) writePacket(data []byte) error { // Handshake Initialization Packet // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::Handshake -func (mc *mysqlConn) readHandshakePacket() ([]byte, string, error) { - data, err := mc.readPacket() +func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err error) { + data, err = mc.readPacket() if err != nil { // for init we can rewrite this to ErrBadConn for sql.Driver to retry, since // in connection initialization we don't risk retrying non-idempotent actions. if err == ErrInvalidConn { return nil, "", driver.ErrBadConn } - return nil, "", err + return } if data[0] == iERR { @@ -198,7 +198,6 @@ func (mc *mysqlConn) readHandshakePacket() ([]byte, string, error) { } pos += 2 - plugin := "" if len(data) > pos { // character set [1 byte] // status flags [2 bytes] @@ -236,8 +235,6 @@ func (mc *mysqlConn) readHandshakePacket() ([]byte, string, error) { return b[:], plugin, nil } - plugin = defaultAuthPlugin - // make a memory safe copy of the cipher slice var b [8]byte copy(b[:], authData) From 447ae1fd2f87c59f657f5c495f3db2612256717e Mon Sep 17 00:00:00 2001 From: Thomas Wodarek Date: Wed, 18 Jul 2018 10:34:48 -0400 Subject: [PATCH 02/93] Update README.md to remove 1.7 support doc (#829) * Update README.md to remove 1.7 support doc Per commit https://github.com/go-sql-driver/mysql/commit/749ddf1598b47e3cd909414bda735fe790ef3d30, 1.7 support was removed, so this will fix the docs to reflect that change. * Add myself to AUTHORS --- AUTHORS | 1 + README.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 73ff68fbc..fbe4ec442 100644 --- a/AUTHORS +++ b/AUTHORS @@ -72,6 +72,7 @@ Shuode Li Soroush Pour Stan Putrya Stanley Gunawan +Thomas Wodarek Xiangyu Hu Xiaobing Jiang Xiuming Chen diff --git a/README.md b/README.md index 2e9b07eeb..7e7df1a3d 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) pac * Optional placeholder interpolation ## Requirements - * Go 1.7 or higher. We aim to support the 3 latest versions of Go. + * Go 1.8 or higher. We aim to support the 3 latest versions of Go. * MySQL (4.1+), MariaDB, Percona Server, Google CloudSQL or Sphinx (2.2.3+) --------------------------------------- From 99ff426eb706cffe92ff3d058e168b278cabf7c7 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 19 Jul 2018 16:19:42 +0900 Subject: [PATCH 03/93] Fix caching_sha2_password with empty password (#826) There shouldn't be trailing NUL byte for caching_sha2_password. Fixes #825 --- auth.go | 2 +- auth_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/auth.go b/auth.go index 0b59f52ee..2f61ecd4f 100644 --- a/auth.go +++ b/auth.go @@ -241,7 +241,7 @@ func (mc *mysqlConn) auth(authData []byte, plugin string) ([]byte, bool, error) switch plugin { case "caching_sha2_password": authResp := scrambleSHA256Password(authData, mc.cfg.Passwd) - return authResp, (authResp == nil), nil + return authResp, false, nil case "mysql_old_password": if !mc.cfg.AllowOldPasswords { diff --git a/auth_test.go b/auth_test.go index 407363be4..bd0e2189c 100644 --- a/auth_test.go +++ b/auth_test.go @@ -764,7 +764,7 @@ func TestAuthSwitchCachingSHA256PasswordEmpty(t *testing.T) { t.Errorf("got error: %v", err) } - expectedReply := []byte{1, 0, 0, 3, 0} + expectedReply := []byte{0, 0, 0, 3} if !bytes.Equal(conn.written, expectedReply) { t.Errorf("got unexpected data: %v", conn.written) } From 361f66ef3b53de1f16b7f2af9ef38a6c159ceb3e Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 1 Oct 2018 16:22:39 +0900 Subject: [PATCH 04/93] Fix cancelled context breaked mysqlConn (#862) Fix #858 --- connection.go | 15 +++++++-------- connection_test.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/connection.go b/connection.go index 911be2060..f74235519 100644 --- a/connection.go +++ b/connection.go @@ -595,22 +595,21 @@ func (mc *mysqlConn) watchCancel(ctx context.Context) error { mc.cleanup() return nil } + // When ctx is already cancelled, don't watch it. + if err := ctx.Err(); err != nil { + return err + } + // When ctx is not cancellable, don't watch it. if ctx.Done() == nil { return nil } - - mc.watching = true - select { - default: - case <-ctx.Done(): - return ctx.Err() - } + // When watcher is not alive, can't watch it. if mc.watcher == nil { return nil } + mc.watching = true mc.watcher <- ctx - return nil } diff --git a/connection_test.go b/connection_test.go index dec376117..352c54ed7 100644 --- a/connection_test.go +++ b/connection_test.go @@ -9,6 +9,7 @@ package mysql import ( + "context" "database/sql/driver" "testing" ) @@ -79,3 +80,31 @@ func TestCheckNamedValue(t *testing.T) { t.Fatalf("uint64 high-bit not converted, got %#v %T", value.Value, value.Value) } } + +// TestCleanCancel tests passed context is cancelled at start. +// No packet should be sent. Connection should keep current status. +func TestCleanCancel(t *testing.T) { + mc := &mysqlConn{ + closech: make(chan struct{}), + } + mc.startWatcher() + defer mc.cleanup() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + for i := 0; i < 3; i++ { // Repeat same behavior + err := mc.Ping(ctx) + if err != context.Canceled { + t.Errorf("expected context.Canceled, got %#v", err) + } + + if mc.closed.IsSet() { + t.Error("expected mc is not closed, closed actually") + } + + if mc.watching { + t.Error("expected watching is false, but true") + } + } +} From 0f257fc7d30f6ce2b1e115390594d96c36c7dd5c Mon Sep 17 00:00:00 2001 From: Ilia Cimpoes Date: Fri, 19 Oct 2018 18:40:22 +0300 Subject: [PATCH 05/93] Call markBadConn in Ping method (#875) * Call markBadConn in Ping method * Add myself to AUTHORS --- AUTHORS | 1 + connection.go | 2 +- connection_test.go | 47 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index fbe4ec442..3f322e802 100644 --- a/AUTHORS +++ b/AUTHORS @@ -35,6 +35,7 @@ Hanno Braun Henri Yandell Hirotaka Yamamoto ICHINOSE Shogo +Ilia Cimpoes INADA Naoki Jacek Szwec James Harr diff --git a/connection.go b/connection.go index f74235519..12a382268 100644 --- a/connection.go +++ b/connection.go @@ -475,7 +475,7 @@ func (mc *mysqlConn) Ping(ctx context.Context) (err error) { defer mc.finish() if err = mc.writeCommandPacket(comPing); err != nil { - return + return mc.markBadConn(err) } return mc.readResultOK() diff --git a/connection_test.go b/connection_test.go index 352c54ed7..2a1c8e888 100644 --- a/connection_test.go +++ b/connection_test.go @@ -11,6 +11,8 @@ package mysql import ( "context" "database/sql/driver" + "errors" + "net" "testing" ) @@ -108,3 +110,48 @@ func TestCleanCancel(t *testing.T) { } } } + +func TestPingMarkBadConnection(t *testing.T) { + nc := badConnection{err: errors.New("boom")} + ms := &mysqlConn{ + netConn: nc, + buf: newBuffer(nc), + maxAllowedPacket: defaultMaxAllowedPacket, + } + + err := ms.Ping(context.Background()) + + if err != driver.ErrBadConn { + t.Errorf("expected driver.ErrBadConn, got %#v", err) + } +} + +func TestPingErrInvalidConn(t *testing.T) { + nc := badConnection{err: errors.New("failed to write"), n: 10} + ms := &mysqlConn{ + netConn: nc, + buf: newBuffer(nc), + maxAllowedPacket: defaultMaxAllowedPacket, + closech: make(chan struct{}), + } + + err := ms.Ping(context.Background()) + + if err != ErrInvalidConn { + t.Errorf("expected ErrInvalidConn, got %#v", err) + } +} + +type badConnection struct { + n int + err error + net.Conn +} + +func (bc badConnection) Write(b []byte) (n int, err error) { + return bc.n, bc.err +} + +func (bc badConnection) Close() error { + return nil +} From 7daee5be2a29576f4bada53f3754b8cf886772a6 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 20 Oct 2018 22:49:31 +0900 Subject: [PATCH 06/93] Fix OldAuthSwitchRequest support (#870) --- auth_test.go | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++ packets.go | 2 +- 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/auth_test.go b/auth_test.go index bd0e2189c..2bfc181ab 100644 --- a/auth_test.go +++ b/auth_test.go @@ -1064,6 +1064,22 @@ func TestAuthSwitchOldPasswordNotAllowed(t *testing.T) { } } +// Same to TestAuthSwitchOldPasswordNotAllowed, but use OldAuthSwitch request. +func TestOldAuthSwitchNotAllowed(t *testing.T) { + conn, mc := newRWMockConn(2) + + // OldAuthSwitch request + conn.data = []byte{1, 0, 0, 2, 0xfe} + conn.maxReads = 1 + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + err := mc.handleAuthResult(authData, plugin) + if err != ErrOldPassword { + t.Errorf("expected ErrOldPassword, got %v", err) + } +} + func TestAuthSwitchOldPassword(t *testing.T) { conn, mc := newRWMockConn(2) mc.cfg.AllowOldPasswords = true @@ -1092,6 +1108,32 @@ func TestAuthSwitchOldPassword(t *testing.T) { } } +// Same to TestAuthSwitchOldPassword, but use OldAuthSwitch request. +func TestOldAuthSwitch(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowOldPasswords = true + mc.cfg.Passwd = "secret" + + // OldAuthSwitch request + conn.data = []byte{1, 0, 0, 2, 0xfe} + + // auth response + conn.queuedReplies = [][]byte{{8, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{9, 0, 0, 3, 86, 83, 83, 79, 74, 78, 65, 66, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} func TestAuthSwitchOldPasswordEmpty(t *testing.T) { conn, mc := newRWMockConn(2) mc.cfg.AllowOldPasswords = true @@ -1120,6 +1162,33 @@ func TestAuthSwitchOldPasswordEmpty(t *testing.T) { } } +// Same to TestAuthSwitchOldPasswordEmpty, but use OldAuthSwitch request. +func TestOldAuthSwitchPasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowOldPasswords = true + mc.cfg.Passwd = "" + + // OldAuthSwitch request. + conn.data = []byte{1, 0, 0, 2, 0xfe} + + // auth response + conn.queuedReplies = [][]byte{{8, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{1, 0, 0, 3, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + func TestAuthSwitchSHA256PasswordEmpty(t *testing.T) { conn, mc := newRWMockConn(2) mc.cfg.Passwd = "" diff --git a/packets.go b/packets.go index 170aaa02b..14aab59e3 100644 --- a/packets.go +++ b/packets.go @@ -479,7 +479,7 @@ func (mc *mysqlConn) readAuthResult() ([]byte, string, error) { return data[1:], "", err case iEOF: - if len(data) < 1 { + if len(data) == 1 { // https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::OldAuthSwitchRequest return nil, "mysql_old_password", nil } From dea3c7b6f9f291f62983aaea85250de9773b4f9f Mon Sep 17 00:00:00 2001 From: Harald Nordgren Date: Fri, 26 Oct 2018 09:10:16 +0200 Subject: [PATCH 07/93] travis: Add go 1.11.x (#876) --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 47dd289a0..75505f144 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ go: - 1.8.x - 1.9.x - 1.10.x + - 1.11.x - master before_install: From 64cea2f07fbecc522a3719e6659e1d50749aad99 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 30 Oct 2018 22:01:07 +0900 Subject: [PATCH 08/93] Remove mysqlContext (#886) It is not needed anymore because we dropped Go 1.7 support --- connection.go | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/connection.go b/connection.go index 12a382268..20aa50910 100644 --- a/connection.go +++ b/connection.go @@ -19,16 +19,6 @@ import ( "time" ) -// a copy of context.Context for Go 1.7 and earlier -type mysqlContext interface { - Done() <-chan struct{} - Err() error - - // defined in context.Context, but not used in this driver: - // Deadline() (deadline time.Time, ok bool) - // Value(key interface{}) interface{} -} - type mysqlConn struct { buf buffer netConn net.Conn @@ -45,7 +35,7 @@ type mysqlConn struct { // for context support (Go 1.8+) watching bool - watcher chan<- mysqlContext + watcher chan<- context.Context closech chan struct{} finished chan<- struct{} canceled atomicError // set non-nil if conn is canceled @@ -614,13 +604,13 @@ func (mc *mysqlConn) watchCancel(ctx context.Context) error { } func (mc *mysqlConn) startWatcher() { - watcher := make(chan mysqlContext, 1) + watcher := make(chan context.Context, 1) mc.watcher = watcher finished := make(chan struct{}) mc.finished = finished go func() { for { - var ctx mysqlContext + var ctx context.Context select { case ctx = <-watcher: case <-mc.closech: From fd197cdcfae0c686792ef5399f223f5b66a30b6b Mon Sep 17 00:00:00 2001 From: Tom Jenkinson Date: Wed, 31 Oct 2018 15:07:16 +0100 Subject: [PATCH 09/93] Return ErrBadConn for temporary Dial error (#867) When `Dial()` returned error and it's `Timeout() == true`, return ErrBadConn to database/sql retry new connection. --- AUTHORS | 1 + driver.go | 4 ++++ driver_test.go | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/AUTHORS b/AUTHORS index 3f322e802..806b5b121 100644 --- a/AUTHORS +++ b/AUTHORS @@ -74,6 +74,7 @@ Soroush Pour Stan Putrya Stanley Gunawan Thomas Wodarek +Tom Jenkinson Xiangyu Hu Xiaobing Jiang Xiuming Chen diff --git a/driver.go b/driver.go index ba1297825..eeb83df01 100644 --- a/driver.go +++ b/driver.go @@ -77,6 +77,10 @@ func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { mc.netConn, err = nd.Dial(mc.cfg.Net, mc.cfg.Addr) } if err != nil { + if nerr, ok := err.(net.Error); ok && nerr.Temporary() { + errLog.Print("net.Error from Dial()': ", nerr.Error()) + return nil, driver.ErrBadConn + } return nil, err } diff --git a/driver_test.go b/driver_test.go index f2bf344e5..cec4b5867 100644 --- a/driver_test.go +++ b/driver_test.go @@ -85,6 +85,23 @@ type DBTest struct { db *sql.DB } +type netErrorMock struct { + temporary bool + timeout bool +} + +func (e netErrorMock) Temporary() bool { + return e.temporary +} + +func (e netErrorMock) Timeout() bool { + return e.timeout +} + +func (e netErrorMock) Error() string { + return fmt.Sprintf("mock net error. Temporary: %v, Timeout %v", e.temporary, e.timeout) +} + func runTestsWithMultiStatement(t *testing.T, dsn string, tests ...func(dbt *DBTest)) { if !available { t.Skipf("MySQL server not running on %s", netAddr) @@ -1801,6 +1818,38 @@ func TestConcurrent(t *testing.T) { }) } +func testDialError(t *testing.T, dialErr error, expectErr error) { + RegisterDial("mydial", func(addr string) (net.Conn, error) { + return nil, dialErr + }) + + db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@mydial(%s)/%s?timeout=30s", user, pass, addr, dbname)) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + _, err = db.Exec("DO 1") + if err != expectErr { + t.Fatalf("was expecting %s. Got: %s", dialErr, err) + } +} + +func TestDialUnknownError(t *testing.T) { + testErr := fmt.Errorf("test") + testDialError(t, testErr, testErr) +} + +func TestDialNonRetryableNetErr(t *testing.T) { + testErr := netErrorMock{} + testDialError(t, testErr, testErr) +} + +func TestDialTemporaryNetErr(t *testing.T) { + testErr := netErrorMock{temporary: true} + testDialError(t, testErr, driver.ErrBadConn) +} + // Tests custom dial functions func TestCustomDial(t *testing.T) { if !available { From 369b5d6e5e8e108ed4ae2f2b1607d444b3807dfb Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 13 Nov 2018 11:38:49 +0900 Subject: [PATCH 10/93] Fix Auth Resnponse packet for cleartext password (#887) Trailing NUL char should be in `string[n] auth-response`. But NUL was after auth-response. --- auth.go | 36 +++++++++++++------------- auth_test.go | 73 ++++++++++++++++++++++++++-------------------------- driver.go | 6 ++--- packets.go | 24 +++++------------ 4 files changed, 64 insertions(+), 75 deletions(-) diff --git a/auth.go b/auth.go index 2f61ecd4f..14f678a87 100644 --- a/auth.go +++ b/auth.go @@ -234,64 +234,64 @@ func (mc *mysqlConn) sendEncryptedPassword(seed []byte, pub *rsa.PublicKey) erro if err != nil { return err } - return mc.writeAuthSwitchPacket(enc, false) + return mc.writeAuthSwitchPacket(enc) } -func (mc *mysqlConn) auth(authData []byte, plugin string) ([]byte, bool, error) { +func (mc *mysqlConn) auth(authData []byte, plugin string) ([]byte, error) { switch plugin { case "caching_sha2_password": authResp := scrambleSHA256Password(authData, mc.cfg.Passwd) - return authResp, false, nil + return authResp, nil case "mysql_old_password": if !mc.cfg.AllowOldPasswords { - return nil, false, ErrOldPassword + return nil, ErrOldPassword } // Note: there are edge cases where this should work but doesn't; // this is currently "wontfix": // https://github.com/go-sql-driver/mysql/issues/184 - authResp := scrambleOldPassword(authData[:8], mc.cfg.Passwd) - return authResp, true, nil + authResp := append(scrambleOldPassword(authData[:8], mc.cfg.Passwd), 0) + return authResp, nil case "mysql_clear_password": if !mc.cfg.AllowCleartextPasswords { - return nil, false, ErrCleartextPassword + return nil, ErrCleartextPassword } // http://dev.mysql.com/doc/refman/5.7/en/cleartext-authentication-plugin.html // http://dev.mysql.com/doc/refman/5.7/en/pam-authentication-plugin.html - return []byte(mc.cfg.Passwd), true, nil + return append([]byte(mc.cfg.Passwd), 0), nil case "mysql_native_password": if !mc.cfg.AllowNativePasswords { - return nil, false, ErrNativePassword + return nil, ErrNativePassword } // https://dev.mysql.com/doc/internals/en/secure-password-authentication.html // Native password authentication only need and will need 20-byte challenge. authResp := scramblePassword(authData[:20], mc.cfg.Passwd) - return authResp, false, nil + return authResp, nil case "sha256_password": if len(mc.cfg.Passwd) == 0 { - return nil, true, nil + return []byte{0}, nil } if mc.cfg.tls != nil || mc.cfg.Net == "unix" { // write cleartext auth packet - return []byte(mc.cfg.Passwd), true, nil + return append([]byte(mc.cfg.Passwd), 0), nil } pubKey := mc.cfg.pubKey if pubKey == nil { // request public key from server - return []byte{1}, false, nil + return []byte{1}, nil } // encrypted password enc, err := encryptPassword(mc.cfg.Passwd, authData, pubKey) - return enc, false, err + return enc, err default: errLog.Print("unknown auth plugin:", plugin) - return nil, false, ErrUnknownPlugin + return nil, ErrUnknownPlugin } } @@ -315,11 +315,11 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { plugin = newPlugin - authResp, addNUL, err := mc.auth(authData, plugin) + authResp, err := mc.auth(authData, plugin) if err != nil { return err } - if err = mc.writeAuthSwitchPacket(authResp, addNUL); err != nil { + if err = mc.writeAuthSwitchPacket(authResp); err != nil { return err } @@ -352,7 +352,7 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { case cachingSha2PasswordPerformFullAuthentication: if mc.cfg.tls != nil || mc.cfg.Net == "unix" { // write cleartext auth packet - err = mc.writeAuthSwitchPacket([]byte(mc.cfg.Passwd), true) + err = mc.writeAuthSwitchPacket(append([]byte(mc.cfg.Passwd), 0)) if err != nil { return err } diff --git a/auth_test.go b/auth_test.go index 2bfc181ab..1920ef39f 100644 --- a/auth_test.go +++ b/auth_test.go @@ -85,11 +85,11 @@ func TestAuthFastCachingSHA256PasswordCached(t *testing.T) { plugin := "caching_sha2_password" // Send Client Authentication Packet - authResp, addNUL, err := mc.auth(authData, plugin) + authResp, err := mc.auth(authData, plugin) if err != nil { t.Fatal(err) } - err = mc.writeHandshakeResponsePacket(authResp, addNUL, plugin) + err = mc.writeHandshakeResponsePacket(authResp, plugin) if err != nil { t.Fatal(err) } @@ -130,11 +130,11 @@ func TestAuthFastCachingSHA256PasswordEmpty(t *testing.T) { plugin := "caching_sha2_password" // Send Client Authentication Packet - authResp, addNUL, err := mc.auth(authData, plugin) + authResp, err := mc.auth(authData, plugin) if err != nil { t.Fatal(err) } - err = mc.writeHandshakeResponsePacket(authResp, addNUL, plugin) + err = mc.writeHandshakeResponsePacket(authResp, plugin) if err != nil { t.Fatal(err) } @@ -172,11 +172,11 @@ func TestAuthFastCachingSHA256PasswordFullRSA(t *testing.T) { plugin := "caching_sha2_password" // Send Client Authentication Packet - authResp, addNUL, err := mc.auth(authData, plugin) + authResp, err := mc.auth(authData, plugin) if err != nil { t.Fatal(err) } - err = mc.writeHandshakeResponsePacket(authResp, addNUL, plugin) + err = mc.writeHandshakeResponsePacket(authResp, plugin) if err != nil { t.Fatal(err) } @@ -228,11 +228,11 @@ func TestAuthFastCachingSHA256PasswordFullRSAWithKey(t *testing.T) { plugin := "caching_sha2_password" // Send Client Authentication Packet - authResp, addNUL, err := mc.auth(authData, plugin) + authResp, err := mc.auth(authData, plugin) if err != nil { t.Fatal(err) } - err = mc.writeHandshakeResponsePacket(authResp, addNUL, plugin) + err = mc.writeHandshakeResponsePacket(authResp, plugin) if err != nil { t.Fatal(err) } @@ -280,11 +280,11 @@ func TestAuthFastCachingSHA256PasswordFullSecure(t *testing.T) { plugin := "caching_sha2_password" // Send Client Authentication Packet - authResp, addNUL, err := mc.auth(authData, plugin) + authResp, err := mc.auth(authData, plugin) if err != nil { t.Fatal(err) } - err = mc.writeHandshakeResponsePacket(authResp, addNUL, plugin) + err = mc.writeHandshakeResponsePacket(authResp, plugin) if err != nil { t.Fatal(err) } @@ -336,7 +336,7 @@ func TestAuthFastCleartextPasswordNotAllowed(t *testing.T) { plugin := "mysql_clear_password" // Send Client Authentication Packet - _, _, err := mc.auth(authData, plugin) + _, err := mc.auth(authData, plugin) if err != ErrCleartextPassword { t.Errorf("expected ErrCleartextPassword, got %v", err) } @@ -353,11 +353,11 @@ func TestAuthFastCleartextPassword(t *testing.T) { plugin := "mysql_clear_password" // Send Client Authentication Packet - authResp, addNUL, err := mc.auth(authData, plugin) + authResp, err := mc.auth(authData, plugin) if err != nil { t.Fatal(err) } - err = mc.writeHandshakeResponsePacket(authResp, addNUL, plugin) + err = mc.writeHandshakeResponsePacket(authResp, plugin) if err != nil { t.Fatal(err) } @@ -367,8 +367,8 @@ func TestAuthFastCleartextPassword(t *testing.T) { authRespEnd := authRespStart + 1 + len(authResp) writtenAuthRespLen := conn.written[authRespStart] writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] - expectedAuthResp := []byte{115, 101, 99, 114, 101, 116} - if writtenAuthRespLen != 6 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + expectedAuthResp := []byte{115, 101, 99, 114, 101, 116, 0} + if writtenAuthRespLen != 7 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) } conn.written = nil @@ -396,11 +396,11 @@ func TestAuthFastCleartextPasswordEmpty(t *testing.T) { plugin := "mysql_clear_password" // Send Client Authentication Packet - authResp, addNUL, err := mc.auth(authData, plugin) + authResp, err := mc.auth(authData, plugin) if err != nil { t.Fatal(err) } - err = mc.writeHandshakeResponsePacket(authResp, addNUL, plugin) + err = mc.writeHandshakeResponsePacket(authResp, plugin) if err != nil { t.Fatal(err) } @@ -410,9 +410,9 @@ func TestAuthFastCleartextPasswordEmpty(t *testing.T) { authRespEnd := authRespStart + 1 + len(authResp) writtenAuthRespLen := conn.written[authRespStart] writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] - if writtenAuthRespLen != 0 { - t.Fatalf("unexpected written auth response (%d bytes): %v", - writtenAuthRespLen, writtenAuthResp) + expectedAuthResp := []byte{0} + if writtenAuthRespLen != 1 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) } conn.written = nil @@ -439,7 +439,7 @@ func TestAuthFastNativePasswordNotAllowed(t *testing.T) { plugin := "mysql_native_password" // Send Client Authentication Packet - _, _, err := mc.auth(authData, plugin) + _, err := mc.auth(authData, plugin) if err != ErrNativePassword { t.Errorf("expected ErrNativePassword, got %v", err) } @@ -455,11 +455,11 @@ func TestAuthFastNativePassword(t *testing.T) { plugin := "mysql_native_password" // Send Client Authentication Packet - authResp, addNUL, err := mc.auth(authData, plugin) + authResp, err := mc.auth(authData, plugin) if err != nil { t.Fatal(err) } - err = mc.writeHandshakeResponsePacket(authResp, addNUL, plugin) + err = mc.writeHandshakeResponsePacket(authResp, plugin) if err != nil { t.Fatal(err) } @@ -498,11 +498,11 @@ func TestAuthFastNativePasswordEmpty(t *testing.T) { plugin := "mysql_native_password" // Send Client Authentication Packet - authResp, addNUL, err := mc.auth(authData, plugin) + authResp, err := mc.auth(authData, plugin) if err != nil { t.Fatal(err) } - err = mc.writeHandshakeResponsePacket(authResp, addNUL, plugin) + err = mc.writeHandshakeResponsePacket(authResp, plugin) if err != nil { t.Fatal(err) } @@ -540,11 +540,11 @@ func TestAuthFastSHA256PasswordEmpty(t *testing.T) { plugin := "sha256_password" // Send Client Authentication Packet - authResp, addNUL, err := mc.auth(authData, plugin) + authResp, err := mc.auth(authData, plugin) if err != nil { t.Fatal(err) } - err = mc.writeHandshakeResponsePacket(authResp, addNUL, plugin) + err = mc.writeHandshakeResponsePacket(authResp, plugin) if err != nil { t.Fatal(err) } @@ -554,7 +554,8 @@ func TestAuthFastSHA256PasswordEmpty(t *testing.T) { authRespEnd := authRespStart + 1 + len(authResp) writtenAuthRespLen := conn.written[authRespStart] writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] - if writtenAuthRespLen != 0 { + expectedAuthResp := []byte{0} + if writtenAuthRespLen != 1 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) } conn.written = nil @@ -587,11 +588,11 @@ func TestAuthFastSHA256PasswordRSA(t *testing.T) { plugin := "sha256_password" // Send Client Authentication Packet - authResp, addNUL, err := mc.auth(authData, plugin) + authResp, err := mc.auth(authData, plugin) if err != nil { t.Fatal(err) } - err = mc.writeHandshakeResponsePacket(authResp, addNUL, plugin) + err = mc.writeHandshakeResponsePacket(authResp, plugin) if err != nil { t.Fatal(err) } @@ -636,11 +637,11 @@ func TestAuthFastSHA256PasswordRSAWithKey(t *testing.T) { plugin := "sha256_password" // Send Client Authentication Packet - authResp, addNUL, err := mc.auth(authData, plugin) + authResp, err := mc.auth(authData, plugin) if err != nil { t.Fatal(err) } - err = mc.writeHandshakeResponsePacket(authResp, addNUL, plugin) + err = mc.writeHandshakeResponsePacket(authResp, plugin) if err != nil { t.Fatal(err) } @@ -669,7 +670,7 @@ func TestAuthFastSHA256PasswordSecure(t *testing.T) { plugin := "sha256_password" // send Client Authentication Packet - authResp, addNUL, err := mc.auth(authData, plugin) + authResp, err := mc.auth(authData, plugin) if err != nil { t.Fatal(err) } @@ -677,18 +678,18 @@ func TestAuthFastSHA256PasswordSecure(t *testing.T) { // unset TLS config to prevent the actual establishment of a TLS wrapper mc.cfg.tls = nil - err = mc.writeHandshakeResponsePacket(authResp, addNUL, plugin) + err = mc.writeHandshakeResponsePacket(authResp, plugin) if err != nil { t.Fatal(err) } // check written auth response authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 - authRespEnd := authRespStart + 1 + len(authResp) + 1 + authRespEnd := authRespStart + 1 + len(authResp) writtenAuthRespLen := conn.written[authRespStart] writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] expectedAuthResp := []byte{115, 101, 99, 114, 101, 116, 0} - if writtenAuthRespLen != 6 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + if writtenAuthRespLen != 7 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) } conn.written = nil diff --git a/driver.go b/driver.go index eeb83df01..67126355c 100644 --- a/driver.go +++ b/driver.go @@ -114,18 +114,18 @@ func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { } // Send Client Authentication Packet - authResp, addNUL, err := mc.auth(authData, plugin) + authResp, err := mc.auth(authData, plugin) if err != nil { // try the default auth plugin, if using the requested plugin failed errLog.Print("could not use requested auth plugin '"+plugin+"': ", err.Error()) plugin = defaultAuthPlugin - authResp, addNUL, err = mc.auth(authData, plugin) + authResp, err = mc.auth(authData, plugin) if err != nil { mc.cleanup() return nil, err } } - if err = mc.writeHandshakeResponsePacket(authResp, addNUL, plugin); err != nil { + if err = mc.writeHandshakeResponsePacket(authResp, plugin); err != nil { mc.cleanup() return nil, err } diff --git a/packets.go b/packets.go index 14aab59e3..9ed640850 100644 --- a/packets.go +++ b/packets.go @@ -243,7 +243,7 @@ func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err erro // Client Authentication Packet // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::HandshakeResponse -func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, plugin string) error { +func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string) error { // Adjust client flags based on server support clientFlags := clientProtocol41 | clientSecureConn | @@ -269,7 +269,8 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, // encode length of the auth plugin data var authRespLEIBuf [9]byte - authRespLEI := appendLengthEncodedInteger(authRespLEIBuf[:0], uint64(len(authResp))) + authRespLen := len(authResp) + authRespLEI := appendLengthEncodedInteger(authRespLEIBuf[:0], uint64(authRespLen)) if len(authRespLEI) > 1 { // if the length can not be written in 1 byte, it must be written as a // length encoded integer @@ -277,9 +278,6 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, } pktLen := 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + len(authRespLEI) + len(authResp) + 21 + 1 - if addNUL { - pktLen++ - } // To specify a db name if n := len(mc.cfg.DBName); n > 0 { @@ -350,10 +348,6 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, // Auth Data [length encoded integer] pos += copy(data[pos:], authRespLEI) pos += copy(data[pos:], authResp) - if addNUL { - data[pos] = 0x00 - pos++ - } // Databasename [null terminated string] if len(mc.cfg.DBName) > 0 { @@ -364,17 +358,15 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, pos += copy(data[pos:], plugin) data[pos] = 0x00 + pos++ // Send Auth packet - return mc.writePacket(data) + return mc.writePacket(data[:pos]) } // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse -func (mc *mysqlConn) writeAuthSwitchPacket(authData []byte, addNUL bool) error { +func (mc *mysqlConn) writeAuthSwitchPacket(authData []byte) error { pktLen := 4 + len(authData) - if addNUL { - pktLen++ - } data := mc.buf.takeSmallBuffer(pktLen) if data == nil { // cannot take the buffer. Something must be wrong with the connection @@ -384,10 +376,6 @@ func (mc *mysqlConn) writeAuthSwitchPacket(authData []byte, addNUL bool) error { // Add the auth data [EOF] copy(data[4:], authData) - if addNUL { - data[pktLen-1] = 0x00 - } - return mc.writePacket(data) } From 6be42e0ff99645d7d9626d779001a46e39c5f280 Mon Sep 17 00:00:00 2001 From: Steven Hartland Date: Fri, 16 Nov 2018 10:20:33 +0000 Subject: [PATCH 11/93] Improve buffer handling (#890) * Eliminate redundant size test in takeBuffer. * Change buffer takeXXX functions to return an error to make it explicit that they can fail. * Add missing error check in handleAuthResult. * Add buffer.store(..) method which can be used by external buffer consumers to update the raw buffer. * Fix some typos and unnecessary UTF-8 characters in comments. * Improve buffer function docs. * Add comments to explain some non-obvious behavior around buffer handling. --- AUTHORS | 2 ++ auth.go | 8 +++++--- buffer.go | 49 +++++++++++++++++++++++++++++----------------- connection.go | 6 +++--- driver.go | 2 +- packets.go | 54 ++++++++++++++++++++++++++++----------------------- 6 files changed, 72 insertions(+), 49 deletions(-) diff --git a/AUTHORS b/AUTHORS index 806b5b121..5ce4f7eca 100644 --- a/AUTHORS +++ b/AUTHORS @@ -73,6 +73,7 @@ Shuode Li Soroush Pour Stan Putrya Stanley Gunawan +Steven Hartland Thomas Wodarek Tom Jenkinson Xiangyu Hu @@ -90,3 +91,4 @@ Keybase Inc. Percona LLC Pivotal Inc. Stripe Inc. +Multiplay Ltd. diff --git a/auth.go b/auth.go index 14f678a87..fec7040d4 100644 --- a/auth.go +++ b/auth.go @@ -360,13 +360,15 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { pubKey := mc.cfg.pubKey if pubKey == nil { // request public key from server - data := mc.buf.takeSmallBuffer(4 + 1) + data, err := mc.buf.takeSmallBuffer(4 + 1) + if err != nil { + return err + } data[4] = cachingSha2PasswordRequestPublicKey mc.writePacket(data) // parse public key - data, err := mc.readPacket() - if err != nil { + if data, err = mc.readPacket(); err != nil { return err } diff --git a/buffer.go b/buffer.go index eb4748bf4..19486bd6f 100644 --- a/buffer.go +++ b/buffer.go @@ -22,17 +22,17 @@ const defaultBufSize = 4096 // The buffer is similar to bufio.Reader / Writer but zero-copy-ish // Also highly optimized for this particular use case. type buffer struct { - buf []byte + buf []byte // buf is a byte buffer who's length and capacity are equal. nc net.Conn idx int length int timeout time.Duration } +// newBuffer allocates and returns a new buffer. func newBuffer(nc net.Conn) buffer { - var b [defaultBufSize]byte return buffer{ - buf: b[:], + buf: make([]byte, defaultBufSize), nc: nc, } } @@ -105,43 +105,56 @@ func (b *buffer) readNext(need int) ([]byte, error) { return b.buf[offset:b.idx], nil } -// returns a buffer with the requested size. +// takeBuffer returns a buffer with the requested size. // If possible, a slice from the existing buffer is returned. // Otherwise a bigger buffer is made. // Only one buffer (total) can be used at a time. -func (b *buffer) takeBuffer(length int) []byte { +func (b *buffer) takeBuffer(length int) ([]byte, error) { if b.length > 0 { - return nil + return nil, ErrBusyBuffer } // test (cheap) general case first - if length <= defaultBufSize || length <= cap(b.buf) { - return b.buf[:length] + if length <= cap(b.buf) { + return b.buf[:length], nil } if length < maxPacketSize { b.buf = make([]byte, length) - return b.buf + return b.buf, nil } - return make([]byte, length) + + // buffer is larger than we want to store. + return make([]byte, length), nil } -// shortcut which can be used if the requested buffer is guaranteed to be -// smaller than defaultBufSize +// takeSmallBuffer is shortcut which can be used if length is +// known to be smaller than defaultBufSize. // Only one buffer (total) can be used at a time. -func (b *buffer) takeSmallBuffer(length int) []byte { +func (b *buffer) takeSmallBuffer(length int) ([]byte, error) { if b.length > 0 { - return nil + return nil, ErrBusyBuffer } - return b.buf[:length] + return b.buf[:length], nil } // takeCompleteBuffer returns the complete existing buffer. // This can be used if the necessary buffer size is unknown. +// cap and len of the returned buffer will be equal. // Only one buffer (total) can be used at a time. -func (b *buffer) takeCompleteBuffer() []byte { +func (b *buffer) takeCompleteBuffer() ([]byte, error) { + if b.length > 0 { + return nil, ErrBusyBuffer + } + return b.buf, nil +} + +// store stores buf, an updated buffer, if its suitable to do so. +func (b *buffer) store(buf []byte) error { if b.length > 0 { - return nil + return ErrBusyBuffer + } else if cap(buf) <= maxPacketSize && cap(buf) > cap(b.buf) { + b.buf = buf[:cap(buf)] } - return b.buf + return nil } diff --git a/connection.go b/connection.go index 20aa50910..fc4ec7597 100644 --- a/connection.go +++ b/connection.go @@ -182,10 +182,10 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin return "", driver.ErrSkip } - buf := mc.buf.takeCompleteBuffer() - if buf == nil { + buf, err := mc.buf.takeCompleteBuffer() + if err != nil { // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return "", ErrInvalidConn } buf = buf[:0] diff --git a/driver.go b/driver.go index 67126355c..9f4967087 100644 --- a/driver.go +++ b/driver.go @@ -50,7 +50,7 @@ func RegisterDial(net string, dial DialFunc) { // Open new Connection. // See https://github.com/go-sql-driver/mysql#dsn-data-source-name for how -// the DSN string is formated +// the DSN string is formatted func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { var err error diff --git a/packets.go b/packets.go index 9ed640850..cfcfff360 100644 --- a/packets.go +++ b/packets.go @@ -51,7 +51,7 @@ func (mc *mysqlConn) readPacket() ([]byte, error) { mc.sequence++ // packets with length 0 terminate a previous packet which is a - // multiple of (2^24)−1 bytes long + // multiple of (2^24)-1 bytes long if pktLen == 0 { // there was no previous packet if prevData == nil { @@ -286,10 +286,10 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string } // Calculate packet length and get buffer with that size - data := mc.buf.takeSmallBuffer(pktLen + 4) - if data == nil { + data, err := mc.buf.takeSmallBuffer(pktLen + 4) + if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return errBadConnNoWrite } @@ -367,10 +367,10 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse func (mc *mysqlConn) writeAuthSwitchPacket(authData []byte) error { pktLen := 4 + len(authData) - data := mc.buf.takeSmallBuffer(pktLen) - if data == nil { + data, err := mc.buf.takeSmallBuffer(pktLen) + if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return errBadConnNoWrite } @@ -387,10 +387,10 @@ func (mc *mysqlConn) writeCommandPacket(command byte) error { // Reset Packet Sequence mc.sequence = 0 - data := mc.buf.takeSmallBuffer(4 + 1) - if data == nil { + data, err := mc.buf.takeSmallBuffer(4 + 1) + if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return errBadConnNoWrite } @@ -406,10 +406,10 @@ func (mc *mysqlConn) writeCommandPacketStr(command byte, arg string) error { mc.sequence = 0 pktLen := 1 + len(arg) - data := mc.buf.takeBuffer(pktLen + 4) - if data == nil { + data, err := mc.buf.takeBuffer(pktLen + 4) + if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return errBadConnNoWrite } @@ -427,10 +427,10 @@ func (mc *mysqlConn) writeCommandPacketUint32(command byte, arg uint32) error { // Reset Packet Sequence mc.sequence = 0 - data := mc.buf.takeSmallBuffer(4 + 1 + 4) - if data == nil { + data, err := mc.buf.takeSmallBuffer(4 + 1 + 4) + if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return errBadConnNoWrite } @@ -883,7 +883,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { const minPktLen = 4 + 1 + 4 + 1 + 4 mc := stmt.mc - // Determine threshould dynamically to avoid packet size shortage. + // Determine threshold dynamically to avoid packet size shortage. longDataSize := mc.maxAllowedPacket / (stmt.paramCount + 1) if longDataSize < 64 { longDataSize = 64 @@ -893,15 +893,17 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { mc.sequence = 0 var data []byte + var err error if len(args) == 0 { - data = mc.buf.takeBuffer(minPktLen) + data, err = mc.buf.takeBuffer(minPktLen) } else { - data = mc.buf.takeCompleteBuffer() + data, err = mc.buf.takeCompleteBuffer() + // In this case the len(data) == cap(data) which is used to optimise the flow below. } - if data == nil { + if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return errBadConnNoWrite } @@ -927,7 +929,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { pos := minPktLen var nullMask []byte - if maskLen, typesLen := (len(args)+7)/8, 1+2*len(args); pos+maskLen+typesLen >= len(data) { + if maskLen, typesLen := (len(args)+7)/8, 1+2*len(args); pos+maskLen+typesLen >= cap(data) { // buffer has to be extended but we don't know by how much so // we depend on append after all data with known sizes fit. // We stop at that because we deal with a lot of columns here @@ -936,10 +938,11 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { copy(tmp[:pos], data[:pos]) data = tmp nullMask = data[pos : pos+maskLen] + // No need to clean nullMask as make ensures that. pos += maskLen } else { nullMask = data[pos : pos+maskLen] - for i := 0; i < maskLen; i++ { + for i := range nullMask { nullMask[i] = 0 } pos += maskLen @@ -1076,7 +1079,10 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { // In that case we must build the data packet with the new values buffer if valuesCap != cap(paramValues) { data = append(data[:pos], paramValues...) - mc.buf.buf = data + if err = mc.buf.store(data); err != nil { + errLog.Print(err) + return errBadConnNoWrite + } } pos += len(paramValues) From 60d456a402782453be397030407e34decaf04d73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Eeden?= Date: Sun, 2 Dec 2018 18:10:36 +0100 Subject: [PATCH 12/93] Implement support of Optional TLS (#900) Issue: #899 Add `preferred` config value to the `tls` config variable on the DSN. This results in a TLS connection when the server advertises this by the flag send in the initial packet. --- README.md | 4 ++-- driver_test.go | 18 +++++++++++++----- dsn.go | 2 +- packets.go | 6 +++++- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 7e7df1a3d..6e4816e63 100644 --- a/README.md +++ b/README.md @@ -328,11 +328,11 @@ Timeout for establishing connections, aka dial timeout. The value must be a deci ``` Type: bool / string -Valid Values: true, false, skip-verify, +Valid Values: true, false, skip-verify, preferred, Default: false ``` -`tls=true` enables TLS / SSL encrypted connection to the server. Use `skip-verify` if you want to use a self-signed or invalid certificate (server side). Use a custom value registered with [`mysql.RegisterTLSConfig`](https://godoc.org/github.com/go-sql-driver/mysql#RegisterTLSConfig). +`tls=true` enables TLS / SSL encrypted connection to the server. Use `skip-verify` if you want to use a self-signed or invalid certificate (server side). Use `preferred` to use TLS only when advertised by the server, this is similar to `skip-verify`, but additionally allows a fallback to a connection which is not encrypted. Use a custom value registered with [`mysql.RegisterTLSConfig`](https://godoc.org/github.com/go-sql-driver/mysql#RegisterTLSConfig). ##### `writeTimeout` diff --git a/driver_test.go b/driver_test.go index cec4b5867..46d1f7ff4 100644 --- a/driver_test.go +++ b/driver_test.go @@ -1304,7 +1304,7 @@ func TestFoundRows(t *testing.T) { } func TestTLS(t *testing.T) { - tlsTest := func(dbt *DBTest) { + tlsTestReq := func(dbt *DBTest) { if err := dbt.db.Ping(); err != nil { if err == ErrNoTLS { dbt.Skip("server does not support TLS") @@ -1321,19 +1321,27 @@ func TestTLS(t *testing.T) { dbt.Fatal(err.Error()) } - if value == nil { - dbt.Fatal("no Cipher") + if (*value == nil) || (len(*value) == 0) { + dbt.Fatalf("no Cipher") + } else { + dbt.Logf("Cipher: %s", *value) } } } + tlsTestOpt := func(dbt *DBTest) { + if err := dbt.db.Ping(); err != nil { + dbt.Fatalf("error on Ping: %s", err.Error()) + } + } - runTests(t, dsn+"&tls=skip-verify", tlsTest) + runTests(t, dsn+"&tls=preferred", tlsTestOpt) + runTests(t, dsn+"&tls=skip-verify", tlsTestReq) // Verify that registering / using a custom cfg works RegisterTLSConfig("custom-skip-verify", &tls.Config{ InsecureSkipVerify: true, }) - runTests(t, dsn+"&tls=custom-skip-verify", tlsTest) + runTests(t, dsn+"&tls=custom-skip-verify", tlsTestReq) } func TestReuseClosedConnection(t *testing.T) { diff --git a/dsn.go b/dsn.go index be014babe..b9134722e 100644 --- a/dsn.go +++ b/dsn.go @@ -560,7 +560,7 @@ func parseDSNParams(cfg *Config, params string) (err error) { } else { cfg.TLSConfig = "false" } - } else if vl := strings.ToLower(value); vl == "skip-verify" { + } else if vl := strings.ToLower(value); vl == "skip-verify" || vl == "preferred" { cfg.TLSConfig = vl cfg.tls = &tls.Config{InsecureSkipVerify: true} } else { diff --git a/packets.go b/packets.go index cfcfff360..5e0853767 100644 --- a/packets.go +++ b/packets.go @@ -194,7 +194,11 @@ func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err erro return nil, "", ErrOldProtocol } if mc.flags&clientSSL == 0 && mc.cfg.tls != nil { - return nil, "", ErrNoTLS + if mc.cfg.TLSConfig == "preferred" { + mc.cfg.tls = nil + } else { + return nil, "", ErrNoTLS + } } pos += 2 From c45f530f8e7fe40f4687eaa50d0c8c5f1b66f9e0 Mon Sep 17 00:00:00 2001 From: Julien Schmidt Date: Tue, 18 Dec 2018 13:36:37 +0100 Subject: [PATCH 13/93] README: warning about insecure TLS configs (#901) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6e4816e63..341d9194c 100644 --- a/README.md +++ b/README.md @@ -332,7 +332,7 @@ Valid Values: true, false, skip-verify, preferred, Default: false ``` -`tls=true` enables TLS / SSL encrypted connection to the server. Use `skip-verify` if you want to use a self-signed or invalid certificate (server side). Use `preferred` to use TLS only when advertised by the server, this is similar to `skip-verify`, but additionally allows a fallback to a connection which is not encrypted. Use a custom value registered with [`mysql.RegisterTLSConfig`](https://godoc.org/github.com/go-sql-driver/mysql#RegisterTLSConfig). +`tls=true` enables TLS / SSL encrypted connection to the server. Use `skip-verify` if you want to use a self-signed or invalid certificate (server side) or use `preferred` to use TLS only when advertised by the server. This is similar to `skip-verify`, but additionally allows a fallback to a connection which is not encrypted. Neither `skip-verify` nor `preferred` add any reliable security. You can use a custom TLS config after registering it with [`mysql.RegisterTLSConfig`](https://godoc.org/github.com/go-sql-driver/mysql#RegisterTLSConfig). ##### `writeTimeout` From 1b9eda25029c206f36637dde8e2acad26df99dab Mon Sep 17 00:00:00 2001 From: Jerome Meyer Date: Sat, 26 Jan 2019 20:11:11 -0500 Subject: [PATCH 14/93] test: close rows (#918) --- AUTHORS | 1 + driver_test.go | 23 ++++++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 5ce4f7eca..c89852c4d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -41,6 +41,7 @@ Jacek Szwec James Harr Jeff Hodges Jeffrey Charles +Jerome Meyer Jian Zhen Joshua Prunier Julien Lefevre diff --git a/driver_test.go b/driver_test.go index 46d1f7ff4..c35588a09 100644 --- a/driver_test.go +++ b/driver_test.go @@ -212,6 +212,7 @@ func TestEmptyQuery(t *testing.T) { runTests(t, dsn, func(dbt *DBTest) { // just a comment, no query rows := dbt.mustQuery("--") + defer rows.Close() // will hang before #255 if rows.Next() { dbt.Errorf("next on rows must be false") @@ -230,6 +231,7 @@ func TestCRUD(t *testing.T) { if rows.Next() { dbt.Error("unexpected data in empty table") } + rows.Close() // Create Data res := dbt.mustExec("INSERT INTO test VALUES (1)") @@ -263,6 +265,7 @@ func TestCRUD(t *testing.T) { } else { dbt.Error("no data") } + rows.Close() // Update res = dbt.mustExec("UPDATE test SET value = ? WHERE value = ?", false, true) @@ -288,6 +291,7 @@ func TestCRUD(t *testing.T) { } else { dbt.Error("no data") } + rows.Close() // Delete res = dbt.mustExec("DELETE FROM test WHERE value = ?", false) @@ -351,6 +355,7 @@ func TestMultiQuery(t *testing.T) { } else { dbt.Error("no data") } + rows.Close() }) } @@ -377,6 +382,7 @@ func TestInt(t *testing.T) { } else { dbt.Errorf("%s: no data", v) } + rows.Close() dbt.mustExec("DROP TABLE IF EXISTS test") } @@ -396,6 +402,7 @@ func TestInt(t *testing.T) { } else { dbt.Errorf("%s ZEROFILL: no data", v) } + rows.Close() dbt.mustExec("DROP TABLE IF EXISTS test") } @@ -420,6 +427,7 @@ func TestFloat32(t *testing.T) { } else { dbt.Errorf("%s: no data", v) } + rows.Close() dbt.mustExec("DROP TABLE IF EXISTS test") } }) @@ -443,6 +451,7 @@ func TestFloat64(t *testing.T) { } else { dbt.Errorf("%s: no data", v) } + rows.Close() dbt.mustExec("DROP TABLE IF EXISTS test") } }) @@ -466,6 +475,7 @@ func TestFloat64Placeholder(t *testing.T) { } else { dbt.Errorf("%s: no data", v) } + rows.Close() dbt.mustExec("DROP TABLE IF EXISTS test") } }) @@ -492,6 +502,7 @@ func TestString(t *testing.T) { } else { dbt.Errorf("%s: no data", v) } + rows.Close() dbt.mustExec("DROP TABLE IF EXISTS test") } @@ -524,6 +535,7 @@ func TestRawBytes(t *testing.T) { v1 := []byte("aaa") v2 := []byte("bbb") rows := dbt.mustQuery("SELECT ?, ?", v1, v2) + defer rows.Close() if rows.Next() { var o1, o2 sql.RawBytes if err := rows.Scan(&o1, &o2); err != nil { @@ -572,6 +584,7 @@ func TestValuer(t *testing.T) { } else { dbt.Errorf("Valuer: no data") } + rows.Close() dbt.mustExec("DROP TABLE IF EXISTS test") }) @@ -884,6 +897,7 @@ func TestTimestampMicros(t *testing.T) { dbt.mustExec("INSERT INTO test SET value0=?, value1=?, value6=?", f0, f1, f6) var res0, res1, res6 string rows := dbt.mustQuery("SELECT * FROM test") + defer rows.Close() if !rows.Next() { dbt.Errorf("test contained no selectable values") } @@ -1042,6 +1056,7 @@ func TestNULL(t *testing.T) { var out interface{} rows := dbt.mustQuery("SELECT * FROM test") + defer rows.Close() if rows.Next() { rows.Scan(&out) if out != nil { @@ -1121,6 +1136,7 @@ func TestLongData(t *testing.T) { inS := in[:maxAllowedPacketSize-nonDataQueryLen] dbt.mustExec("INSERT INTO test VALUES('" + inS + "')") rows = dbt.mustQuery("SELECT value FROM test") + defer rows.Close() if rows.Next() { rows.Scan(&out) if inS != out { @@ -1139,6 +1155,7 @@ func TestLongData(t *testing.T) { // Long binary data dbt.mustExec("INSERT INTO test VALUES(?)", in) rows = dbt.mustQuery("SELECT value FROM test WHERE 1=?", 1) + defer rows.Close() if rows.Next() { rows.Scan(&out) if in != out { @@ -1314,6 +1331,7 @@ func TestTLS(t *testing.T) { } rows := dbt.mustQuery("SHOW STATUS LIKE 'Ssl_cipher'") + defer rows.Close() var variable, value *sql.RawBytes for rows.Next() { @@ -1474,9 +1492,9 @@ func TestColumnsWithAlias(t *testing.T) { if cols[0] != "A" { t.Fatalf("expected column name \"A\", got \"%s\"", cols[0]) } - rows.Close() rows = dbt.mustQuery("SELECT * FROM (SELECT 1 AS one) AS A") + defer rows.Close() cols, _ = rows.Columns() if len(cols) != 1 { t.Fatalf("expected 1 column, got %d", len(cols)) @@ -1520,6 +1538,7 @@ func TestTimezoneConversion(t *testing.T) { // Retrieve time from DB rows := dbt.mustQuery("SELECT ts FROM test") + defer rows.Close() if !rows.Next() { dbt.Fatal("did not get any rows out") } @@ -2017,6 +2036,7 @@ func TestInterruptBySignal(t *testing.T) { dbt.Errorf("expected val to be 42") } } + rows.Close() // binary protocol rows, err = dbt.db.Query("CALL test_signal(?)", 42) @@ -2030,6 +2050,7 @@ func TestInterruptBySignal(t *testing.T) { dbt.Errorf("expected val to be 42") } } + rows.Close() }) } From 972a708cf97995463843c08c8585b26997daf0e1 Mon Sep 17 00:00:00 2001 From: Simon J Mudd Date: Sun, 17 Feb 2019 08:26:58 +0100 Subject: [PATCH 15/93] Utils typo wether --> whether. (#895) --- AUTHORS | 1 + utils.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index c89852c4d..5482a8536 100644 --- a/AUTHORS +++ b/AUTHORS @@ -71,6 +71,7 @@ Richard Wilkes Robert Russell Runrioter Wung Shuode Li +Simon J Mudd Soroush Pour Stan Putrya Stanley Gunawan diff --git a/utils.go b/utils.go index cb3650bb9..201691fe8 100644 --- a/utils.go +++ b/utils.go @@ -684,7 +684,7 @@ type atomicBool struct { value uint32 } -// IsSet returns wether the current boolean value is true +// IsSet returns whether the current boolean value is true func (ab *atomicBool) IsSet() bool { return atomic.LoadUint32(&ab.value) > 0 } @@ -698,7 +698,7 @@ func (ab *atomicBool) Set(value bool) { } } -// TrySet sets the value of the bool and returns wether the value changed +// TrySet sets the value of the bool and returns whether the value changed func (ab *atomicBool) TrySet(value bool) bool { if value { return atomic.SwapUint32(&ab.value, 1) == 0 From 2c9d54fefcfb443251f5b92cd5106cf4dc07b880 Mon Sep 17 00:00:00 2001 From: Tim Ruffles Date: Thu, 7 Mar 2019 23:26:31 -0600 Subject: [PATCH 16/93] README: 'Equivalent' is clearer (#927) * Equivalent is clearer * Update AUTHORS --- AUTHORS | 1 + README.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 5482a8536..146cdffdd 100644 --- a/AUTHORS +++ b/AUTHORS @@ -77,6 +77,7 @@ Stan Putrya Stanley Gunawan Steven Hartland Thomas Wodarek +Tim Ruffles Tom Jenkinson Xiangyu Hu Xiaobing Jiang diff --git a/README.md b/README.md index 341d9194c..999ff4c2c 100644 --- a/README.md +++ b/README.md @@ -444,7 +444,7 @@ See the [godoc of Go-MySQL-Driver](https://godoc.org/github.com/go-sql-driver/my ### `time.Time` support The default internal output type of MySQL `DATE` and `DATETIME` values is `[]byte` which allows you to scan the value into a `[]byte`, `string` or `sql.RawBytes` variable in your program. -However, many want to scan MySQL `DATE` and `DATETIME` values into `time.Time` variables, which is the logical opposite in Go to `DATE` and `DATETIME` in MySQL. You can do that by changing the internal output type from `[]byte` to `time.Time` with the DSN parameter `parseTime=true`. You can set the default [`time.Time` location](https://golang.org/pkg/time/#Location) with the `loc` DSN parameter. +However, many want to scan MySQL `DATE` and `DATETIME` values into `time.Time` variables, which is the logical equivalent in Go to `DATE` and `DATETIME` in MySQL. You can do that by changing the internal output type from `[]byte` to `time.Time` with the DSN parameter `parseTime=true`. You can set the default [`time.Time` location](https://golang.org/pkg/time/#Location) with the `loc` DSN parameter. **Caution:** As of Go 1.1, this makes `time.Time` the only variable type you can scan `DATE` and `DATETIME` values into. This breaks for example [`sql.RawBytes` support](https://github.com/go-sql-driver/mysql/wiki/Examples#rawbytes). From 1fbca2aabb09915d4f1006d01b271ad7778eec4f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 16 Mar 2019 07:32:36 +0900 Subject: [PATCH 17/93] drop Go 1.8 support (#936) * drop Go 1.8 support * travis: add Go 1.12.x --- .travis.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 75505f144..8f653936b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,10 @@ sudo: false language: go go: - - 1.8.x - 1.9.x - 1.10.x - 1.11.x + - 1.12.x - master before_install: diff --git a/README.md b/README.md index 999ff4c2c..7b435be7b 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) pac * Optional placeholder interpolation ## Requirements - * Go 1.8 or higher. We aim to support the 3 latest versions of Go. + * Go 1.9 or higher. We aim to support the 3 latest versions of Go. * MySQL (4.1+), MariaDB, Percona Server, Google CloudSQL or Sphinx (2.2.3+) --------------------------------------- From bc5e6eaa6d4fc8a7c8668050c7b1813afeafcabe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicent=20Mart=C3=AD?= Date: Fri, 29 Mar 2019 09:29:56 +0100 Subject: [PATCH 18/93] check connection liveness before writing query (#934) This commit contains a potential fix to the issue reported in #657. As a summary: when a MySQL server kills a connection on the server-side (either because it is actively pruning connections, or because the connection has hit the server-side timeout), the Go MySQL client does not immediately become aware of the connection being dead. Because of the way TCP works, the client cannot know that the connection has received a RST packet from the server (i.e. the server-side has closed) until it actually reads from it. This causes an unfortunate bug wherein a MySQL idle connection is pulled from the connection pool, a query packet is written to it without error, and then the query fails with an "unexpected EOF" error when trying to read the response packet. Since the initial write to the socket does not fail with an error, it is generally not safe to return `driver.ErrBadConn` when the read fails, because in theory the write could have arrived to the server and could have been committed. Returning `ErrBadConn` could lead to duplicate inserts on the database and data corruption because of the way the Go SQL package performs retries. In order to significantly reduce the circumstances where this "unexpected EOF" error is returned for stale connections, this commit performs a liveness check before writing a new query. When do we check? ----------------- This check is not performed for all writes. Go 1.10 introduced a new `sql/driver` interface called `driver.SessionResetter`, which calls the `ResetSession` method on any connections _when they are returned to the connection pool_. Since performing the liveness check during `ResetSession` is not particularly useful (the connection can spend a long time in the pool before it's checked out again, and become stale), we simply mark the connection with a `reset` flag instead. This `reset` flag is then checked from `mysqlConn.writePacket` to perform the liveness checks. This ensures that the liveness check will only be performed for the first query on a connection that has been checked out of the connection pool. These are pretty much the semantics we want: a fresh connection from the pool is more likely to be stale, and it has not performed any previous writes that could cause data corruption. If a connection is being consistently used by the client (i.e. through an open transaction), we do NOT perform liveness checks. If MySQL Server kills such active connection, we want to bubble up the error to the user because any silent retrying can and will lead to data corruption. Since the `ResetSession` interface is only available in Go 1.10+, the liveness checks will only be performed starting with that Go version. How do we check? ---------------- To perform the actual liveness test on the connection, we use the new `syscall.Conn` interface which is available for all `net.Conn`s since Go 1.9. The `SyscallConn` method returns a `RawConn` that lets us read directly from the connection's file descriptor using syscalls, and skipping the default read pipeline of the Go runtime. When reading directly from the file descriptor using `syscall.Read`, we pass in a 1-length buffer, as passing a 0-length buffer will always result in a 0-length read, and the 1-length buffer will never be filled because we're not expecting any reads from MySQL before we have written any request packets in a fresh connection. All sockets created in the Go runtime are set to non-blocking (O_NONBLOCK). Consequently, we can detect a socket that has been closed on the server-side because the `read` syscall will return a 0-length read _and_ no error. We assume that any other errors returned from the `read` also mean the connection is in a bad state, except for `EAGAIN`/`EWOULDBLOCK`, which is the expected return for a healthy non-blocking socket in this circumstance. Because of the dependency on `syscall.Conn`, liveness checks can only be performed in Go 1.9+. This restriction however overlaps with the fact that we only mark connections as having been reset in Go 1.10+, as explained in the previous section. --- AUTHORS | 1 + conncheck.go | 53 ++++++++++++++++++++++++++++++++++++++++++++ conncheck_test.go | 38 +++++++++++++++++++++++++++++++ conncheck_windows.go | 15 +++++++++++++ connection.go | 3 +++ packets.go | 20 +++++++++++++++++ 6 files changed, 130 insertions(+) create mode 100644 conncheck.go create mode 100644 conncheck_test.go create mode 100644 conncheck_windows.go diff --git a/AUTHORS b/AUTHORS index 146cdffdd..fb6668346 100644 --- a/AUTHORS +++ b/AUTHORS @@ -88,6 +88,7 @@ Zhenye Xie Barracuda Networks, Inc. Counting Ltd. +GitHub Inc. Google Inc. InfoSum Ltd. Keybase Inc. diff --git a/conncheck.go b/conncheck.go new file mode 100644 index 000000000..fa868e84d --- /dev/null +++ b/conncheck.go @@ -0,0 +1,53 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2019 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build !windows + +package mysql + +import ( + "errors" + "io" + "net" + "syscall" +) + +var errUnexpectedRead = errors.New("unexpected read from socket") + +func connCheck(c net.Conn) error { + var ( + n int + err error + buff [1]byte + ) + + sconn, ok := c.(syscall.Conn) + if !ok { + return nil + } + rc, err := sconn.SyscallConn() + if err != nil { + return err + } + rerr := rc.Read(func(fd uintptr) bool { + n, err = syscall.Read(int(fd), buff[:]) + return true + }) + switch { + case rerr != nil: + return rerr + case n == 0 && err == nil: + return io.EOF + case n > 0: + return errUnexpectedRead + case err == syscall.EAGAIN || err == syscall.EWOULDBLOCK: + return nil + default: + return err + } +} diff --git a/conncheck_test.go b/conncheck_test.go new file mode 100644 index 000000000..b7234b0f5 --- /dev/null +++ b/conncheck_test.go @@ -0,0 +1,38 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build go1.10,!windows + +package mysql + +import ( + "testing" + "time" +) + +func TestStaleConnectionChecks(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("SET @@SESSION.wait_timeout = 2") + + if err := dbt.db.Ping(); err != nil { + dbt.Fatal(err) + } + + // wait for MySQL to close our connection + time.Sleep(3 * time.Second) + + tx, err := dbt.db.Begin() + if err != nil { + dbt.Fatal(err) + } + + if err := tx.Rollback(); err != nil { + dbt.Fatal(err) + } + }) +} diff --git a/conncheck_windows.go b/conncheck_windows.go new file mode 100644 index 000000000..3d9e63f66 --- /dev/null +++ b/conncheck_windows.go @@ -0,0 +1,15 @@ +package mysql + +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2019 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +import "net" + +func connCheck(c net.Conn) error { + return nil +} diff --git a/connection.go b/connection.go index fc4ec7597..265fd4e47 100644 --- a/connection.go +++ b/connection.go @@ -22,6 +22,7 @@ import ( type mysqlConn struct { buf buffer netConn net.Conn + rawConn net.Conn // underlying connection when netConn is TLS connection. affectedRows uint64 insertId uint64 cfg *Config @@ -32,6 +33,7 @@ type mysqlConn struct { status statusFlag sequence uint8 parseTime bool + reset bool // set when the Go SQL package calls ResetSession // for context support (Go 1.8+) watching bool @@ -639,5 +641,6 @@ func (mc *mysqlConn) ResetSession(ctx context.Context) error { if mc.closed.IsSet() { return driver.ErrBadConn } + mc.reset = true return nil } diff --git a/packets.go b/packets.go index 5e0853767..70d0d71f5 100644 --- a/packets.go +++ b/packets.go @@ -96,6 +96,25 @@ func (mc *mysqlConn) writePacket(data []byte) error { return ErrPktTooLarge } + // Perform a stale connection check. We only perform this check for + // the first query on a connection that has been checked out of the + // connection pool: a fresh connection from the pool is more likely + // to be stale, and it has not performed any previous writes that + // could cause data corruption, so it's safe to return ErrBadConn + // if the check fails. + if mc.reset { + mc.reset = false + conn := mc.netConn + if mc.rawConn != nil { + conn = mc.rawConn + } + if err := connCheck(conn); err != nil { + errLog.Print("closing bad idle connection: ", err) + mc.Close() + return driver.ErrBadConn + } + } + for { var size int if pktLen >= maxPacketSize { @@ -332,6 +351,7 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string if err := tlsConn.Handshake(); err != nil { return err } + mc.rawConn = mc.netConn mc.netConn = tlsConn mc.buf.nc = tlsConn } From c0f6b444ad8f92ecd68f86b27635c1fccdede4c4 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 30 Mar 2019 12:22:41 +0900 Subject: [PATCH 19/93] travis: add macOS test (#939) --- .travis.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.travis.yml b/.travis.yml index 8f653936b..eae311b14 100644 --- a/.travis.yml +++ b/.travis.yml @@ -99,6 +99,27 @@ matrix: - export MYSQL_TEST_ADDR=127.0.0.1:3307 - export MYSQL_TEST_CONCURRENT=1 + - os: osx + osx_image: xcode10.1 + addons: + homebrew: + packages: + - mysql + go: 1.12.x + before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + before_script: + - echo -e "[server]\ninnodb_log_file_size=256MB\ninnodb_buffer_pool_size=512MB\nmax_allowed_packet=16MB\nlocal_infile=1" >> /usr/local/etc/my.cnf + - mysql.server start + - mysql -uroot -e 'CREATE USER gotest IDENTIFIED BY "secret"' + - mysql -uroot -e 'GRANT ALL ON *.* TO gotest' + - mysql -uroot -e 'create database gotest;' + - export MYSQL_TEST_USER=gotest + - export MYSQL_TEST_PASS=secret + - export MYSQL_TEST_ADDR=127.0.0.1:3306 + - export MYSQL_TEST_CONCURRENT=1 + script: - go test -v -covermode=count -coverprofile=coverage.out - go vet ./... From df597a2343f1ef9dba230c7cf008fc31ca2b5d90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicent=20Mart=C3=AD?= Date: Thu, 4 Apr 2019 12:18:22 +0200 Subject: [PATCH 20/93] buffer: Use a double-buffering scheme to prevent data races (#943) Fixes #903 Co-Authored-By: vmg --- benchmark_test.go | 54 ++++++++++++++++++++++++++++++++++++++++++++++ buffer.go | 48 ++++++++++++++++++++++++++++++----------- driver_test.go | 55 +++++++++++++++++++++++++++++++++++++++++++++++ rows.go | 7 ++++++ 4 files changed, 151 insertions(+), 13 deletions(-) diff --git a/benchmark_test.go b/benchmark_test.go index 5828d40f9..3e25a3bf2 100644 --- a/benchmark_test.go +++ b/benchmark_test.go @@ -317,3 +317,57 @@ func BenchmarkExecContext(b *testing.B) { }) } } + +// BenchmarkQueryRawBytes benchmarks fetching 100 blobs using sql.RawBytes. +// "size=" means size of each blobs. +func BenchmarkQueryRawBytes(b *testing.B) { + var sizes []int = []int{100, 1000, 2000, 4000, 8000, 12000, 16000, 32000, 64000, 256000} + db := initDB(b, + "DROP TABLE IF EXISTS bench_rawbytes", + "CREATE TABLE bench_rawbytes (id INT PRIMARY KEY, val LONGBLOB)", + ) + defer db.Close() + + blob := make([]byte, sizes[len(sizes)-1]) + for i := range blob { + blob[i] = 42 + } + for i := 0; i < 100; i++ { + _, err := db.Exec("INSERT INTO bench_rawbytes VALUES (?, ?)", i, blob) + if err != nil { + b.Fatal(err) + } + } + + for _, s := range sizes { + b.Run(fmt.Sprintf("size=%v", s), func(b *testing.B) { + db.SetMaxIdleConns(0) + db.SetMaxIdleConns(1) + b.ReportAllocs() + b.ResetTimer() + + for j := 0; j < b.N; j++ { + rows, err := db.Query("SELECT LEFT(val, ?) as v FROM bench_rawbytes", s) + if err != nil { + b.Fatal(err) + } + nrows := 0 + for rows.Next() { + var buf sql.RawBytes + err := rows.Scan(&buf) + if err != nil { + b.Fatal(err) + } + if len(buf) != s { + b.Fatalf("size mismatch: expected %v, got %v", s, len(buf)) + } + nrows++ + } + rows.Close() + if nrows != 100 { + b.Fatalf("numbers of rows mismatch: expected %v, got %v", 100, nrows) + } + } + }) + } +} diff --git a/buffer.go b/buffer.go index 19486bd6f..0774c5c8c 100644 --- a/buffer.go +++ b/buffer.go @@ -15,47 +15,69 @@ import ( ) const defaultBufSize = 4096 +const maxCachedBufSize = 256 * 1024 // A buffer which is used for both reading and writing. // This is possible since communication on each connection is synchronous. // In other words, we can't write and read simultaneously on the same connection. // The buffer is similar to bufio.Reader / Writer but zero-copy-ish // Also highly optimized for this particular use case. +// This buffer is backed by two byte slices in a double-buffering scheme type buffer struct { buf []byte // buf is a byte buffer who's length and capacity are equal. nc net.Conn idx int length int timeout time.Duration + dbuf [2][]byte // dbuf is an array with the two byte slices that back this buffer + flipcnt uint // flipccnt is the current buffer counter for double-buffering } // newBuffer allocates and returns a new buffer. func newBuffer(nc net.Conn) buffer { + fg := make([]byte, defaultBufSize) return buffer{ - buf: make([]byte, defaultBufSize), - nc: nc, + buf: fg, + nc: nc, + dbuf: [2][]byte{fg, nil}, } } +// flip replaces the active buffer with the background buffer +// this is a delayed flip that simply increases the buffer counter; +// the actual flip will be performed the next time we call `buffer.fill` +func (b *buffer) flip() { + b.flipcnt += 1 +} + // fill reads into the buffer until at least _need_ bytes are in it func (b *buffer) fill(need int) error { n := b.length + // fill data into its double-buffering target: if we've called + // flip on this buffer, we'll be copying to the background buffer, + // and then filling it with network data; otherwise we'll just move + // the contents of the current buffer to the front before filling it + dest := b.dbuf[b.flipcnt&1] + + // grow buffer if necessary to fit the whole packet. + if need > len(dest) { + // Round up to the next multiple of the default size + dest = make([]byte, ((need/defaultBufSize)+1)*defaultBufSize) - // move existing data to the beginning - if n > 0 && b.idx > 0 { - copy(b.buf[0:n], b.buf[b.idx:]) + // if the allocated buffer is not too large, move it to backing storage + // to prevent extra allocations on applications that perform large reads + if len(dest) <= maxCachedBufSize { + b.dbuf[b.flipcnt&1] = dest + } } - // grow buffer if necessary - // TODO: let the buffer shrink again at some point - // Maybe keep the org buf slice and swap back? - if need > len(b.buf) { - // Round up to the next multiple of the default size - newBuf := make([]byte, ((need/defaultBufSize)+1)*defaultBufSize) - copy(newBuf, b.buf) - b.buf = newBuf + // if we're filling the fg buffer, move the existing data to the start of it. + // if we're filling the bg buffer, copy over the data + if n > 0 { + copy(dest[:n], b.buf[b.idx:]) } + b.buf = dest b.idx = 0 for { diff --git a/driver_test.go b/driver_test.go index c35588a09..9c3d286ce 100644 --- a/driver_test.go +++ b/driver_test.go @@ -2938,3 +2938,58 @@ func TestValuerWithValueReceiverGivenNilValue(t *testing.T) { // This test will panic on the INSERT if ConvertValue() does not check for typed nil before calling Value() }) } + +// TestRawBytesAreNotModified checks for a race condition that arises when a query context +// is canceled while a user is calling rows.Scan. This is a more stringent test than the one +// proposed in https://github.com/golang/go/issues/23519. Here we're explicitly using +// `sql.RawBytes` to check the contents of our internal buffers are not modified after an implicit +// call to `Rows.Close`, so Context cancellation should **not** invalidate the backing buffers. +func TestRawBytesAreNotModified(t *testing.T) { + const blob = "abcdefghijklmnop" + const contextRaceIterations = 20 + const blobSize = defaultBufSize * 3 / 4 // Second row overwrites first row. + const insertRows = 4 + + var sqlBlobs = [2]string{ + strings.Repeat(blob, blobSize/len(blob)), + strings.Repeat(strings.ToUpper(blob), blobSize/len(blob)), + } + + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (id int, value BLOB) CHARACTER SET utf8") + for i := 0; i < insertRows; i++ { + dbt.mustExec("INSERT INTO test VALUES (?, ?)", i+1, sqlBlobs[i&1]) + } + + for i := 0; i < contextRaceIterations; i++ { + func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rows, err := dbt.db.QueryContext(ctx, `SELECT id, value FROM test`) + if err != nil { + t.Fatal(err) + } + + var b int + var raw sql.RawBytes + for rows.Next() { + if err := rows.Scan(&b, &raw); err != nil { + t.Fatal(err) + } + + before := string(raw) + // Ensure cancelling the query does not corrupt the contents of `raw` + cancel() + time.Sleep(time.Microsecond * 100) + after := string(raw) + + if before != after { + t.Fatalf("the backing storage for sql.RawBytes has been modified (i=%v)", i) + } + } + rows.Close() + }() + } + }) +} diff --git a/rows.go b/rows.go index d3b1e2822..888bdb5f0 100644 --- a/rows.go +++ b/rows.go @@ -111,6 +111,13 @@ func (rows *mysqlRows) Close() (err error) { return err } + // flip the buffer for this connection if we need to drain it. + // note that for a successful query (i.e. one where rows.next() + // has been called until it returns false), `rows.mc` will be nil + // by the time the user calls `(*Rows).Close`, so we won't reach this + // see: https://github.com/golang/go/commit/651ddbdb5056ded455f47f9c494c67b389622a47 + mc.buf.flip() + // Remove unread packets from stream if !rows.rs.done { err = mc.readUntilEOF() From 89ec2a9ec85e27afb0f0fcd3c2399e72cc360ae4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicent=20Mart=C3=AD?= Date: Thu, 4 Apr 2019 15:56:37 +0200 Subject: [PATCH 21/93] Support Go 1.10 Connector interface (#941) --- appengine.go | 7 ++- connector.go | 143 +++++++++++++++++++++++++++++++++++++++++++ driver.go | 138 ++++++++--------------------------------- driver_go110.go | 37 +++++++++++ driver_go110_test.go | 137 +++++++++++++++++++++++++++++++++++++++++ driver_test.go | 7 ++- dsn.go | 21 +++++++ dsn_test.go | 40 ++++++++++++ 8 files changed, 415 insertions(+), 115 deletions(-) create mode 100644 connector.go create mode 100644 driver_go110.go create mode 100644 driver_go110_test.go diff --git a/appengine.go b/appengine.go index be41f2ee6..44c0fd7c7 100644 --- a/appengine.go +++ b/appengine.go @@ -11,9 +11,14 @@ package mysql import ( + "context" + "google.golang.org/appengine/cloudsql" ) func init() { - RegisterDial("cloudsql", cloudsql.Dial) + RegisterDialContext("cloudsql", func(_ context.Context, instance addr) (net.Conn, error) { + // XXX: the cloudsql driver still does not export a Context-aware dialer. + return cloudsql.Dial(instance) + }) } diff --git a/connector.go b/connector.go new file mode 100644 index 000000000..5aaaba43e --- /dev/null +++ b/connector.go @@ -0,0 +1,143 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "context" + "database/sql/driver" + "net" +) + +type connector struct { + cfg *Config // immutable private copy. +} + +// Connect implements driver.Connector interface. +// Connect returns a connection to the database. +func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { + var err error + + // New mysqlConn + mc := &mysqlConn{ + maxAllowedPacket: maxPacketSize, + maxWriteSize: maxPacketSize - 1, + closech: make(chan struct{}), + cfg: c.cfg, + } + mc.parseTime = mc.cfg.ParseTime + + // Connect to Server + dialsLock.RLock() + dial, ok := dials[mc.cfg.Net] + dialsLock.RUnlock() + if ok { + mc.netConn, err = dial(ctx, mc.cfg.Addr) + } else { + nd := net.Dialer{Timeout: mc.cfg.Timeout} + mc.netConn, err = nd.DialContext(ctx, mc.cfg.Net, mc.cfg.Addr) + } + + if err != nil { + if nerr, ok := err.(net.Error); ok && nerr.Temporary() { + errLog.Print("net.Error from Dial()': ", nerr.Error()) + return nil, driver.ErrBadConn + } + return nil, err + } + + // Enable TCP Keepalives on TCP connections + if tc, ok := mc.netConn.(*net.TCPConn); ok { + if err := tc.SetKeepAlive(true); err != nil { + // Don't send COM_QUIT before handshake. + mc.netConn.Close() + mc.netConn = nil + return nil, err + } + } + + // Call startWatcher for context support (From Go 1.8) + mc.startWatcher() + if err := mc.watchCancel(ctx); err != nil { + return nil, err + } + defer mc.finish() + + mc.buf = newBuffer(mc.netConn) + + // Set I/O timeouts + mc.buf.timeout = mc.cfg.ReadTimeout + mc.writeTimeout = mc.cfg.WriteTimeout + + // Reading Handshake Initialization Packet + authData, plugin, err := mc.readHandshakePacket() + if err != nil { + mc.cleanup() + return nil, err + } + + if plugin == "" { + plugin = defaultAuthPlugin + } + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + // try the default auth plugin, if using the requested plugin failed + errLog.Print("could not use requested auth plugin '"+plugin+"': ", err.Error()) + plugin = defaultAuthPlugin + authResp, err = mc.auth(authData, plugin) + if err != nil { + mc.cleanup() + return nil, err + } + } + if err = mc.writeHandshakeResponsePacket(authResp, plugin); err != nil { + mc.cleanup() + return nil, err + } + + // Handle response to auth packet, switch methods if possible + if err = mc.handleAuthResult(authData, plugin); err != nil { + // Authentication failed and MySQL has already closed the connection + // (https://dev.mysql.com/doc/internals/en/authentication-fails.html). + // Do not send COM_QUIT, just cleanup and return the error. + mc.cleanup() + return nil, err + } + + if mc.cfg.MaxAllowedPacket > 0 { + mc.maxAllowedPacket = mc.cfg.MaxAllowedPacket + } else { + // Get max allowed packet size + maxap, err := mc.getSystemVar("max_allowed_packet") + if err != nil { + mc.Close() + return nil, err + } + mc.maxAllowedPacket = stringToInt(maxap) - 1 + } + if mc.maxAllowedPacket < maxPacketSize { + mc.maxWriteSize = mc.maxAllowedPacket + } + + // Handle DSN Params + err = mc.handleParams() + if err != nil { + mc.Close() + return nil, err + } + + return mc, nil +} + +// Driver implements driver.Connector interface. +// Driver returns &MySQLDriver{}. +func (c *connector) Driver() driver.Driver { + return &MySQLDriver{} +} diff --git a/driver.go b/driver.go index 9f4967087..1f9decf80 100644 --- a/driver.go +++ b/driver.go @@ -17,6 +17,7 @@ package mysql import ( + "context" "database/sql" "database/sql/driver" "net" @@ -29,139 +30,54 @@ type MySQLDriver struct{} // DialFunc is a function which can be used to establish the network connection. // Custom dial functions must be registered with RegisterDial +// +// Deprecated: users should register a DialContextFunc instead type DialFunc func(addr string) (net.Conn, error) +// DialContextFunc is a function which can be used to establish the network connection. +// Custom dial functions must be registered with RegisterDialContext +type DialContextFunc func(ctx context.Context, addr string) (net.Conn, error) + var ( dialsLock sync.RWMutex - dials map[string]DialFunc + dials map[string]DialContextFunc ) -// RegisterDial registers a custom dial function. It can then be used by the +// RegisterDialContext registers a custom dial function. It can then be used by the // network address mynet(addr), where mynet is the registered new network. -// addr is passed as a parameter to the dial function. -func RegisterDial(net string, dial DialFunc) { +// The current context for the connection and its address is passed to the dial function. +func RegisterDialContext(net string, dial DialContextFunc) { dialsLock.Lock() defer dialsLock.Unlock() if dials == nil { - dials = make(map[string]DialFunc) + dials = make(map[string]DialContextFunc) } dials[net] = dial } +// RegisterDial registers a custom dial function. It can then be used by the +// network address mynet(addr), where mynet is the registered new network. +// addr is passed as a parameter to the dial function. +// +// Deprecated: users should call RegisterDialContext instead +func RegisterDial(network string, dial DialFunc) { + RegisterDialContext(network, func(_ context.Context, addr string) (net.Conn, error) { + return dial(addr) + }) +} + // Open new Connection. // See https://github.com/go-sql-driver/mysql#dsn-data-source-name for how // the DSN string is formatted func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { - var err error - - // New mysqlConn - mc := &mysqlConn{ - maxAllowedPacket: maxPacketSize, - maxWriteSize: maxPacketSize - 1, - closech: make(chan struct{}), - } - mc.cfg, err = ParseDSN(dsn) - if err != nil { - return nil, err - } - mc.parseTime = mc.cfg.ParseTime - - // Connect to Server - dialsLock.RLock() - dial, ok := dials[mc.cfg.Net] - dialsLock.RUnlock() - if ok { - mc.netConn, err = dial(mc.cfg.Addr) - } else { - nd := net.Dialer{Timeout: mc.cfg.Timeout} - mc.netConn, err = nd.Dial(mc.cfg.Net, mc.cfg.Addr) - } - if err != nil { - if nerr, ok := err.(net.Error); ok && nerr.Temporary() { - errLog.Print("net.Error from Dial()': ", nerr.Error()) - return nil, driver.ErrBadConn - } - return nil, err - } - - // Enable TCP Keepalives on TCP connections - if tc, ok := mc.netConn.(*net.TCPConn); ok { - if err := tc.SetKeepAlive(true); err != nil { - // Don't send COM_QUIT before handshake. - mc.netConn.Close() - mc.netConn = nil - return nil, err - } - } - - // Call startWatcher for context support (From Go 1.8) - mc.startWatcher() - - mc.buf = newBuffer(mc.netConn) - - // Set I/O timeouts - mc.buf.timeout = mc.cfg.ReadTimeout - mc.writeTimeout = mc.cfg.WriteTimeout - - // Reading Handshake Initialization Packet - authData, plugin, err := mc.readHandshakePacket() + cfg, err := ParseDSN(dsn) if err != nil { - mc.cleanup() return nil, err } - if plugin == "" { - plugin = defaultAuthPlugin + c := &connector{ + cfg: cfg, } - - // Send Client Authentication Packet - authResp, err := mc.auth(authData, plugin) - if err != nil { - // try the default auth plugin, if using the requested plugin failed - errLog.Print("could not use requested auth plugin '"+plugin+"': ", err.Error()) - plugin = defaultAuthPlugin - authResp, err = mc.auth(authData, plugin) - if err != nil { - mc.cleanup() - return nil, err - } - } - if err = mc.writeHandshakeResponsePacket(authResp, plugin); err != nil { - mc.cleanup() - return nil, err - } - - // Handle response to auth packet, switch methods if possible - if err = mc.handleAuthResult(authData, plugin); err != nil { - // Authentication failed and MySQL has already closed the connection - // (https://dev.mysql.com/doc/internals/en/authentication-fails.html). - // Do not send COM_QUIT, just cleanup and return the error. - mc.cleanup() - return nil, err - } - - if mc.cfg.MaxAllowedPacket > 0 { - mc.maxAllowedPacket = mc.cfg.MaxAllowedPacket - } else { - // Get max allowed packet size - maxap, err := mc.getSystemVar("max_allowed_packet") - if err != nil { - mc.Close() - return nil, err - } - mc.maxAllowedPacket = stringToInt(maxap) - 1 - } - if mc.maxAllowedPacket < maxPacketSize { - mc.maxWriteSize = mc.maxAllowedPacket - } - - // Handle DSN Params - err = mc.handleParams() - if err != nil { - mc.Close() - return nil, err - } - - return mc, nil + return c.Connect(context.Background()) } func init() { diff --git a/driver_go110.go b/driver_go110.go new file mode 100644 index 000000000..eb5a8fe9b --- /dev/null +++ b/driver_go110.go @@ -0,0 +1,37 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build go1.10 + +package mysql + +import ( + "database/sql/driver" +) + +// NewConnector returns new driver.Connector. +func NewConnector(cfg *Config) (driver.Connector, error) { + cfg = cfg.Clone() + // normalize the contents of cfg so calls to NewConnector have the same + // behavior as MySQLDriver.OpenConnector + if err := cfg.normalize(); err != nil { + return nil, err + } + return &connector{cfg: cfg}, nil +} + +// OpenConnector implements driver.DriverContext. +func (d MySQLDriver) OpenConnector(dsn string) (driver.Connector, error) { + cfg, err := ParseDSN(dsn) + if err != nil { + return nil, err + } + return &connector{ + cfg: cfg, + }, nil +} diff --git a/driver_go110_test.go b/driver_go110_test.go new file mode 100644 index 000000000..19a0e5956 --- /dev/null +++ b/driver_go110_test.go @@ -0,0 +1,137 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build go1.10 + +package mysql + +import ( + "context" + "database/sql" + "database/sql/driver" + "fmt" + "net" + "testing" + "time" +) + +var _ driver.DriverContext = &MySQLDriver{} + +type dialCtxKey struct{} + +func TestConnectorObeysDialTimeouts(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + RegisterDialContext("dialctxtest", func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + if !ctx.Value(dialCtxKey{}).(bool) { + return nil, fmt.Errorf("test error: query context is not propagated to our dialer") + } + return d.DialContext(ctx, prot, addr) + }) + + db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@dialctxtest(%s)/%s?timeout=30s", user, pass, addr, dbname)) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + ctx := context.WithValue(context.Background(), dialCtxKey{}, true) + + _, err = db.ExecContext(ctx, "DO 1") + if err != nil { + t.Fatal(err) + } +} + +func configForTests(t *testing.T) *Config { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + mycnf := NewConfig() + mycnf.User = user + mycnf.Passwd = pass + mycnf.Addr = addr + mycnf.Net = prot + mycnf.DBName = dbname + return mycnf +} + +func TestNewConnector(t *testing.T) { + mycnf := configForTests(t) + conn, err := NewConnector(mycnf) + if err != nil { + t.Fatal(err) + } + + db := sql.OpenDB(conn) + defer db.Close() + + if err := db.Ping(); err != nil { + t.Fatal(err) + } +} + +type slowConnection struct { + net.Conn + slowdown time.Duration +} + +func (sc *slowConnection) Read(b []byte) (int, error) { + time.Sleep(sc.slowdown) + return sc.Conn.Read(b) +} + +type connectorHijack struct { + driver.Connector + connErr error +} + +func (cw *connectorHijack) Connect(ctx context.Context) (driver.Conn, error) { + var conn driver.Conn + conn, cw.connErr = cw.Connector.Connect(ctx) + return conn, cw.connErr +} + +func TestConnectorTimeoutsDuringOpen(t *testing.T) { + RegisterDialContext("slowconn", func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + conn, err := d.DialContext(ctx, prot, addr) + if err != nil { + return nil, err + } + return &slowConnection{Conn: conn, slowdown: 100 * time.Millisecond}, nil + }) + + mycnf := configForTests(t) + mycnf.Net = "slowconn" + + conn, err := NewConnector(mycnf) + if err != nil { + t.Fatal(err) + } + + hijack := &connectorHijack{Connector: conn} + + db := sql.OpenDB(hijack) + defer db.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + _, err = db.ExecContext(ctx, "DO 1") + if err != context.DeadlineExceeded { + t.Fatalf("ExecContext should have timed out") + } + if hijack.connErr != context.DeadlineExceeded { + t.Fatalf("(*Connector).Connect should have timed out") + } +} diff --git a/driver_test.go b/driver_test.go index 9c3d286ce..b45a81eb1 100644 --- a/driver_test.go +++ b/driver_test.go @@ -1846,7 +1846,7 @@ func TestConcurrent(t *testing.T) { } func testDialError(t *testing.T, dialErr error, expectErr error) { - RegisterDial("mydial", func(addr string) (net.Conn, error) { + RegisterDialContext("mydial", func(ctx context.Context, addr string) (net.Conn, error) { return nil, dialErr }) @@ -1884,8 +1884,9 @@ func TestCustomDial(t *testing.T) { } // our custom dial function which justs wraps net.Dial here - RegisterDial("mydial", func(addr string) (net.Conn, error) { - return net.Dial(prot, addr) + RegisterDialContext("mydial", func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, prot, addr) }) db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@mydial(%s)/%s?timeout=30s", user, pass, addr, dbname)) diff --git a/dsn.go b/dsn.go index b9134722e..6e19ab717 100644 --- a/dsn.go +++ b/dsn.go @@ -14,6 +14,7 @@ import ( "crypto/tls" "errors" "fmt" + "math/big" "net" "net/url" "sort" @@ -72,6 +73,26 @@ func NewConfig() *Config { } } +func (cfg *Config) Clone() *Config { + cp := *cfg + if cp.tls != nil { + cp.tls = cfg.tls.Clone() + } + if len(cp.Params) > 0 { + cp.Params = make(map[string]string, len(cfg.Params)) + for k, v := range cfg.Params { + cp.Params[k] = v + } + } + if cfg.pubKey != nil { + cp.pubKey = &rsa.PublicKey{ + N: new(big.Int).Set(cfg.pubKey.N), + E: cfg.pubKey.E, + } + } + return &cp +} + func (cfg *Config) normalize() error { if cfg.InterpolateParams && unsafeCollations[cfg.Collation] { return errInvalidDSNUnsafeCollation diff --git a/dsn_test.go b/dsn_test.go index 1cd095496..280fdf61f 100644 --- a/dsn_test.go +++ b/dsn_test.go @@ -318,6 +318,46 @@ func TestParamsAreSorted(t *testing.T) { } } +func TestCloneConfig(t *testing.T) { + RegisterServerPubKey("testKey", testPubKeyRSA) + defer DeregisterServerPubKey("testKey") + + expectedServerName := "example.com" + dsn := "tcp(example.com:1234)/?tls=true&foobar=baz&serverPubKey=testKey" + cfg, err := ParseDSN(dsn) + if err != nil { + t.Fatal(err.Error()) + } + + cfg2 := cfg.Clone() + if cfg == cfg2 { + t.Errorf("Config.Clone did not create a separate config struct") + } + + if cfg2.tls.ServerName != expectedServerName { + t.Errorf("cfg.tls.ServerName should be %q, got %q (host with port)", expectedServerName, cfg.tls.ServerName) + } + + cfg2.tls.ServerName = "example2.com" + if cfg.tls.ServerName == cfg2.tls.ServerName { + t.Errorf("changed cfg.tls.Server name should not propagate to original Config") + } + + if _, ok := cfg2.Params["foobar"]; !ok { + t.Errorf("cloned Config is missing custom params") + } + + delete(cfg2.Params, "foobar") + + if _, ok := cfg.Params["foobar"]; !ok { + t.Errorf("custom params in cloned Config should not propagate to original Config") + } + + if !reflect.DeepEqual(cfg.pubKey, cfg2.pubKey) { + t.Errorf("public key in Config should be identical") + } +} + func BenchmarkParseDSN(b *testing.B) { b.ReportAllocs() From 8f4b98d14697cfd66f2264bfa2277a26426fbaea Mon Sep 17 00:00:00 2001 From: Erwan Martin Date: Fri, 5 Apr 2019 01:10:27 +0200 Subject: [PATCH 22/93] Fix regression in appengine.go (#949) --- AUTHORS | 1 + appengine.go | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index fb6668346..e85106a42 100644 --- a/AUTHORS +++ b/AUTHORS @@ -27,6 +27,7 @@ Daniël van Eeden Dave Protasowski DisposaBoy Egor Smolyakov +Erwan Martin Evan Shaw Frederick Mayle Gustavo Kristic diff --git a/appengine.go b/appengine.go index 44c0fd7c7..914e6623b 100644 --- a/appengine.go +++ b/appengine.go @@ -12,12 +12,13 @@ package mysql import ( "context" + "net" "google.golang.org/appengine/cloudsql" ) func init() { - RegisterDialContext("cloudsql", func(_ context.Context, instance addr) (net.Conn, error) { + RegisterDialContext("cloudsql", func(_ context.Context, instance string) (net.Conn, error) { // XXX: the cloudsql driver still does not export a Context-aware dialer. return cloudsql.Dial(instance) }) From d3a0b0fcd73ce97db48da235e7e17a21228cb186 Mon Sep 17 00:00:00 2001 From: Darren Hoo Date: Fri, 5 Apr 2019 08:15:06 +0800 Subject: [PATCH 23/93] pass unsigned int without converting it to string (#838) --- AUTHORS | 1 + connection_test.go | 4 ++-- packets.go | 16 ++++++++++++++++ statement.go | 11 ++--------- statement_test.go | 6 +++--- 5 files changed, 24 insertions(+), 14 deletions(-) diff --git a/AUTHORS b/AUTHORS index e85106a42..cdf7ae3b8 100644 --- a/AUTHORS +++ b/AUTHORS @@ -35,6 +35,7 @@ Hajime Nakagami Hanno Braun Henri Yandell Hirotaka Yamamoto +Huyiguang ICHINOSE Shogo Ilia Cimpoes INADA Naoki diff --git a/connection_test.go b/connection_test.go index 2a1c8e888..8e78f36c6 100644 --- a/connection_test.go +++ b/connection_test.go @@ -78,8 +78,8 @@ func TestCheckNamedValue(t *testing.T) { t.Fatal("uint64 high-bit not convertible", err) } - if value.Value != "18446744073709551615" { - t.Fatalf("uint64 high-bit not converted, got %#v %T", value.Value, value.Value) + if value.Value != ^uint64(0) { + t.Fatalf("uint64 high-bit converted, got %#v %T", value.Value, value.Value) } } diff --git a/packets.go b/packets.go index 70d0d71f5..cbed325f4 100644 --- a/packets.go +++ b/packets.go @@ -1011,6 +1011,22 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { ) } + case uint64: + paramTypes[i+i] = byte(fieldTypeLongLong) + paramTypes[i+i+1] = 0x80 // type is unsigned + + if cap(paramValues)-len(paramValues)-8 >= 0 { + paramValues = paramValues[:len(paramValues)+8] + binary.LittleEndian.PutUint64( + paramValues[len(paramValues)-8:], + uint64(v), + ) + } else { + paramValues = append(paramValues, + uint64ToBytes(uint64(v))..., + ) + } + case float64: paramTypes[i+i] = byte(fieldTypeDouble) paramTypes[i+i+1] = 0x00 diff --git a/statement.go b/statement.go index ce7fe4cd0..f7e370939 100644 --- a/statement.go +++ b/statement.go @@ -13,7 +13,6 @@ import ( "fmt" "io" "reflect" - "strconv" ) type mysqlStmt struct { @@ -164,14 +163,8 @@ func (c converter) ConvertValue(v interface{}) (driver.Value, error) { } case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: return rv.Int(), nil - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32: - return int64(rv.Uint()), nil - case reflect.Uint64: - u64 := rv.Uint() - if u64 >= 1<<63 { - return strconv.FormatUint(u64, 10), nil - } - return int64(u64), nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return rv.Uint(), nil case reflect.Float32, reflect.Float64: return rv.Float(), nil case reflect.Bool: diff --git a/statement_test.go b/statement_test.go index 98a6c1933..4b9914f8e 100644 --- a/statement_test.go +++ b/statement_test.go @@ -110,7 +110,7 @@ func TestConvertUnsignedIntegers(t *testing.T) { t.Fatalf("%T type not convertible %s", value, err) } - if output != int64(42) { + if output != uint64(42) { t.Fatalf("%T type not converted, got %#v %T", value, output, output) } } @@ -120,7 +120,7 @@ func TestConvertUnsignedIntegers(t *testing.T) { t.Fatal("uint64 high-bit not convertible", err) } - if output != "18446744073709551615" { - t.Fatalf("uint64 high-bit not converted, got %#v %T", output, output) + if output != ^uint64(0) { + t.Fatalf("uint64 high-bit converted, got %#v %T", output, output) } } From 93c3765e9bb2e29c6a63277c9f829dae34ec24eb Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 5 Apr 2019 11:49:13 +0900 Subject: [PATCH 24/93] Update collations and make utf8mb4 default (#877) --- README.md | 7 +- collations.go | 376 +++++++++++++++++++++++++------------------------ driver_test.go | 2 +- dsn_test.go | 32 ++--- 4 files changed, 218 insertions(+), 199 deletions(-) diff --git a/README.md b/README.md index 7b435be7b..c6adf1d63 100644 --- a/README.md +++ b/README.md @@ -171,13 +171,18 @@ Unless you need the fallback behavior, please use `collation` instead. ``` Type: string Valid Values: -Default: utf8_general_ci +Default: utf8mb4_general_ci ``` Sets the collation used for client-server interaction on connection. In contrast to `charset`, `collation` does not issue additional queries. If the specified collation is unavailable on the target server, the connection will fail. A list of valid charsets for a server is retrievable with `SHOW COLLATION`. +The default collation (`utf8mb4_general_ci`) is supported from MySQL 5.5. You should use an older collation (e.g. `utf8_general_ci`) for older MySQL. + +Collations for charset "ucs2", "utf16", "utf16le", and "utf32" can not be used ([ref](https://dev.mysql.com/doc/refman/5.7/en/charset-connection.html#charset-connection-impermissible-client-charset)). + + ##### `clientFoundRows` ``` diff --git a/collations.go b/collations.go index 136c9e4d1..8d2b55676 100644 --- a/collations.go +++ b/collations.go @@ -8,183 +8,190 @@ package mysql -const defaultCollation = "utf8_general_ci" +const defaultCollation = "utf8mb4_general_ci" const binaryCollation = "binary" // A list of available collations mapped to the internal ID. // To update this map use the following MySQL query: -// SELECT COLLATION_NAME, ID FROM information_schema.COLLATIONS +// SELECT COLLATION_NAME, ID FROM information_schema.COLLATIONS WHERE ID<256 ORDER BY ID +// +// Handshake packet have only 1 byte for collation_id. So we can't use collations with ID > 255. +// +// ucs2, utf16, and utf32 can't be used for connection charset. +// https://dev.mysql.com/doc/refman/5.7/en/charset-connection.html#charset-connection-impermissible-client-charset +// They are commented out to reduce this map. var collations = map[string]byte{ - "big5_chinese_ci": 1, - "latin2_czech_cs": 2, - "dec8_swedish_ci": 3, - "cp850_general_ci": 4, - "latin1_german1_ci": 5, - "hp8_english_ci": 6, - "koi8r_general_ci": 7, - "latin1_swedish_ci": 8, - "latin2_general_ci": 9, - "swe7_swedish_ci": 10, - "ascii_general_ci": 11, - "ujis_japanese_ci": 12, - "sjis_japanese_ci": 13, - "cp1251_bulgarian_ci": 14, - "latin1_danish_ci": 15, - "hebrew_general_ci": 16, - "tis620_thai_ci": 18, - "euckr_korean_ci": 19, - "latin7_estonian_cs": 20, - "latin2_hungarian_ci": 21, - "koi8u_general_ci": 22, - "cp1251_ukrainian_ci": 23, - "gb2312_chinese_ci": 24, - "greek_general_ci": 25, - "cp1250_general_ci": 26, - "latin2_croatian_ci": 27, - "gbk_chinese_ci": 28, - "cp1257_lithuanian_ci": 29, - "latin5_turkish_ci": 30, - "latin1_german2_ci": 31, - "armscii8_general_ci": 32, - "utf8_general_ci": 33, - "cp1250_czech_cs": 34, - "ucs2_general_ci": 35, - "cp866_general_ci": 36, - "keybcs2_general_ci": 37, - "macce_general_ci": 38, - "macroman_general_ci": 39, - "cp852_general_ci": 40, - "latin7_general_ci": 41, - "latin7_general_cs": 42, - "macce_bin": 43, - "cp1250_croatian_ci": 44, - "utf8mb4_general_ci": 45, - "utf8mb4_bin": 46, - "latin1_bin": 47, - "latin1_general_ci": 48, - "latin1_general_cs": 49, - "cp1251_bin": 50, - "cp1251_general_ci": 51, - "cp1251_general_cs": 52, - "macroman_bin": 53, - "utf16_general_ci": 54, - "utf16_bin": 55, - "utf16le_general_ci": 56, - "cp1256_general_ci": 57, - "cp1257_bin": 58, - "cp1257_general_ci": 59, - "utf32_general_ci": 60, - "utf32_bin": 61, - "utf16le_bin": 62, - "binary": 63, - "armscii8_bin": 64, - "ascii_bin": 65, - "cp1250_bin": 66, - "cp1256_bin": 67, - "cp866_bin": 68, - "dec8_bin": 69, - "greek_bin": 70, - "hebrew_bin": 71, - "hp8_bin": 72, - "keybcs2_bin": 73, - "koi8r_bin": 74, - "koi8u_bin": 75, - "latin2_bin": 77, - "latin5_bin": 78, - "latin7_bin": 79, - "cp850_bin": 80, - "cp852_bin": 81, - "swe7_bin": 82, - "utf8_bin": 83, - "big5_bin": 84, - "euckr_bin": 85, - "gb2312_bin": 86, - "gbk_bin": 87, - "sjis_bin": 88, - "tis620_bin": 89, - "ucs2_bin": 90, - "ujis_bin": 91, - "geostd8_general_ci": 92, - "geostd8_bin": 93, - "latin1_spanish_ci": 94, - "cp932_japanese_ci": 95, - "cp932_bin": 96, - "eucjpms_japanese_ci": 97, - "eucjpms_bin": 98, - "cp1250_polish_ci": 99, - "utf16_unicode_ci": 101, - "utf16_icelandic_ci": 102, - "utf16_latvian_ci": 103, - "utf16_romanian_ci": 104, - "utf16_slovenian_ci": 105, - "utf16_polish_ci": 106, - "utf16_estonian_ci": 107, - "utf16_spanish_ci": 108, - "utf16_swedish_ci": 109, - "utf16_turkish_ci": 110, - "utf16_czech_ci": 111, - "utf16_danish_ci": 112, - "utf16_lithuanian_ci": 113, - "utf16_slovak_ci": 114, - "utf16_spanish2_ci": 115, - "utf16_roman_ci": 116, - "utf16_persian_ci": 117, - "utf16_esperanto_ci": 118, - "utf16_hungarian_ci": 119, - "utf16_sinhala_ci": 120, - "utf16_german2_ci": 121, - "utf16_croatian_ci": 122, - "utf16_unicode_520_ci": 123, - "utf16_vietnamese_ci": 124, - "ucs2_unicode_ci": 128, - "ucs2_icelandic_ci": 129, - "ucs2_latvian_ci": 130, - "ucs2_romanian_ci": 131, - "ucs2_slovenian_ci": 132, - "ucs2_polish_ci": 133, - "ucs2_estonian_ci": 134, - "ucs2_spanish_ci": 135, - "ucs2_swedish_ci": 136, - "ucs2_turkish_ci": 137, - "ucs2_czech_ci": 138, - "ucs2_danish_ci": 139, - "ucs2_lithuanian_ci": 140, - "ucs2_slovak_ci": 141, - "ucs2_spanish2_ci": 142, - "ucs2_roman_ci": 143, - "ucs2_persian_ci": 144, - "ucs2_esperanto_ci": 145, - "ucs2_hungarian_ci": 146, - "ucs2_sinhala_ci": 147, - "ucs2_german2_ci": 148, - "ucs2_croatian_ci": 149, - "ucs2_unicode_520_ci": 150, - "ucs2_vietnamese_ci": 151, - "ucs2_general_mysql500_ci": 159, - "utf32_unicode_ci": 160, - "utf32_icelandic_ci": 161, - "utf32_latvian_ci": 162, - "utf32_romanian_ci": 163, - "utf32_slovenian_ci": 164, - "utf32_polish_ci": 165, - "utf32_estonian_ci": 166, - "utf32_spanish_ci": 167, - "utf32_swedish_ci": 168, - "utf32_turkish_ci": 169, - "utf32_czech_ci": 170, - "utf32_danish_ci": 171, - "utf32_lithuanian_ci": 172, - "utf32_slovak_ci": 173, - "utf32_spanish2_ci": 174, - "utf32_roman_ci": 175, - "utf32_persian_ci": 176, - "utf32_esperanto_ci": 177, - "utf32_hungarian_ci": 178, - "utf32_sinhala_ci": 179, - "utf32_german2_ci": 180, - "utf32_croatian_ci": 181, - "utf32_unicode_520_ci": 182, - "utf32_vietnamese_ci": 183, + "big5_chinese_ci": 1, + "latin2_czech_cs": 2, + "dec8_swedish_ci": 3, + "cp850_general_ci": 4, + "latin1_german1_ci": 5, + "hp8_english_ci": 6, + "koi8r_general_ci": 7, + "latin1_swedish_ci": 8, + "latin2_general_ci": 9, + "swe7_swedish_ci": 10, + "ascii_general_ci": 11, + "ujis_japanese_ci": 12, + "sjis_japanese_ci": 13, + "cp1251_bulgarian_ci": 14, + "latin1_danish_ci": 15, + "hebrew_general_ci": 16, + "tis620_thai_ci": 18, + "euckr_korean_ci": 19, + "latin7_estonian_cs": 20, + "latin2_hungarian_ci": 21, + "koi8u_general_ci": 22, + "cp1251_ukrainian_ci": 23, + "gb2312_chinese_ci": 24, + "greek_general_ci": 25, + "cp1250_general_ci": 26, + "latin2_croatian_ci": 27, + "gbk_chinese_ci": 28, + "cp1257_lithuanian_ci": 29, + "latin5_turkish_ci": 30, + "latin1_german2_ci": 31, + "armscii8_general_ci": 32, + "utf8_general_ci": 33, + "cp1250_czech_cs": 34, + //"ucs2_general_ci": 35, + "cp866_general_ci": 36, + "keybcs2_general_ci": 37, + "macce_general_ci": 38, + "macroman_general_ci": 39, + "cp852_general_ci": 40, + "latin7_general_ci": 41, + "latin7_general_cs": 42, + "macce_bin": 43, + "cp1250_croatian_ci": 44, + "utf8mb4_general_ci": 45, + "utf8mb4_bin": 46, + "latin1_bin": 47, + "latin1_general_ci": 48, + "latin1_general_cs": 49, + "cp1251_bin": 50, + "cp1251_general_ci": 51, + "cp1251_general_cs": 52, + "macroman_bin": 53, + //"utf16_general_ci": 54, + //"utf16_bin": 55, + //"utf16le_general_ci": 56, + "cp1256_general_ci": 57, + "cp1257_bin": 58, + "cp1257_general_ci": 59, + //"utf32_general_ci": 60, + //"utf32_bin": 61, + //"utf16le_bin": 62, + "binary": 63, + "armscii8_bin": 64, + "ascii_bin": 65, + "cp1250_bin": 66, + "cp1256_bin": 67, + "cp866_bin": 68, + "dec8_bin": 69, + "greek_bin": 70, + "hebrew_bin": 71, + "hp8_bin": 72, + "keybcs2_bin": 73, + "koi8r_bin": 74, + "koi8u_bin": 75, + "utf8_tolower_ci": 76, + "latin2_bin": 77, + "latin5_bin": 78, + "latin7_bin": 79, + "cp850_bin": 80, + "cp852_bin": 81, + "swe7_bin": 82, + "utf8_bin": 83, + "big5_bin": 84, + "euckr_bin": 85, + "gb2312_bin": 86, + "gbk_bin": 87, + "sjis_bin": 88, + "tis620_bin": 89, + //"ucs2_bin": 90, + "ujis_bin": 91, + "geostd8_general_ci": 92, + "geostd8_bin": 93, + "latin1_spanish_ci": 94, + "cp932_japanese_ci": 95, + "cp932_bin": 96, + "eucjpms_japanese_ci": 97, + "eucjpms_bin": 98, + "cp1250_polish_ci": 99, + //"utf16_unicode_ci": 101, + //"utf16_icelandic_ci": 102, + //"utf16_latvian_ci": 103, + //"utf16_romanian_ci": 104, + //"utf16_slovenian_ci": 105, + //"utf16_polish_ci": 106, + //"utf16_estonian_ci": 107, + //"utf16_spanish_ci": 108, + //"utf16_swedish_ci": 109, + //"utf16_turkish_ci": 110, + //"utf16_czech_ci": 111, + //"utf16_danish_ci": 112, + //"utf16_lithuanian_ci": 113, + //"utf16_slovak_ci": 114, + //"utf16_spanish2_ci": 115, + //"utf16_roman_ci": 116, + //"utf16_persian_ci": 117, + //"utf16_esperanto_ci": 118, + //"utf16_hungarian_ci": 119, + //"utf16_sinhala_ci": 120, + //"utf16_german2_ci": 121, + //"utf16_croatian_ci": 122, + //"utf16_unicode_520_ci": 123, + //"utf16_vietnamese_ci": 124, + //"ucs2_unicode_ci": 128, + //"ucs2_icelandic_ci": 129, + //"ucs2_latvian_ci": 130, + //"ucs2_romanian_ci": 131, + //"ucs2_slovenian_ci": 132, + //"ucs2_polish_ci": 133, + //"ucs2_estonian_ci": 134, + //"ucs2_spanish_ci": 135, + //"ucs2_swedish_ci": 136, + //"ucs2_turkish_ci": 137, + //"ucs2_czech_ci": 138, + //"ucs2_danish_ci": 139, + //"ucs2_lithuanian_ci": 140, + //"ucs2_slovak_ci": 141, + //"ucs2_spanish2_ci": 142, + //"ucs2_roman_ci": 143, + //"ucs2_persian_ci": 144, + //"ucs2_esperanto_ci": 145, + //"ucs2_hungarian_ci": 146, + //"ucs2_sinhala_ci": 147, + //"ucs2_german2_ci": 148, + //"ucs2_croatian_ci": 149, + //"ucs2_unicode_520_ci": 150, + //"ucs2_vietnamese_ci": 151, + //"ucs2_general_mysql500_ci": 159, + //"utf32_unicode_ci": 160, + //"utf32_icelandic_ci": 161, + //"utf32_latvian_ci": 162, + //"utf32_romanian_ci": 163, + //"utf32_slovenian_ci": 164, + //"utf32_polish_ci": 165, + //"utf32_estonian_ci": 166, + //"utf32_spanish_ci": 167, + //"utf32_swedish_ci": 168, + //"utf32_turkish_ci": 169, + //"utf32_czech_ci": 170, + //"utf32_danish_ci": 171, + //"utf32_lithuanian_ci": 172, + //"utf32_slovak_ci": 173, + //"utf32_spanish2_ci": 174, + //"utf32_roman_ci": 175, + //"utf32_persian_ci": 176, + //"utf32_esperanto_ci": 177, + //"utf32_hungarian_ci": 178, + //"utf32_sinhala_ci": 179, + //"utf32_german2_ci": 180, + //"utf32_croatian_ci": 181, + //"utf32_unicode_520_ci": 182, + //"utf32_vietnamese_ci": 183, "utf8_unicode_ci": 192, "utf8_icelandic_ci": 193, "utf8_latvian_ci": 194, @@ -234,18 +241,25 @@ var collations = map[string]byte{ "utf8mb4_croatian_ci": 245, "utf8mb4_unicode_520_ci": 246, "utf8mb4_vietnamese_ci": 247, + "gb18030_chinese_ci": 248, + "gb18030_bin": 249, + "gb18030_unicode_520_ci": 250, + "utf8mb4_0900_ai_ci": 255, } // A blacklist of collations which is unsafe to interpolate parameters. // These multibyte encodings may contains 0x5c (`\`) in their trailing bytes. var unsafeCollations = map[string]bool{ - "big5_chinese_ci": true, - "sjis_japanese_ci": true, - "gbk_chinese_ci": true, - "big5_bin": true, - "gb2312_bin": true, - "gbk_bin": true, - "sjis_bin": true, - "cp932_japanese_ci": true, - "cp932_bin": true, + "big5_chinese_ci": true, + "sjis_japanese_ci": true, + "gbk_chinese_ci": true, + "big5_bin": true, + "gb2312_bin": true, + "gbk_bin": true, + "sjis_bin": true, + "cp932_japanese_ci": true, + "cp932_bin": true, + "gb18030_chinese_ci": true, + "gb18030_bin": true, + "gb18030_unicode_520_ci": true, } diff --git a/driver_test.go b/driver_test.go index b45a81eb1..3dee1bab2 100644 --- a/driver_test.go +++ b/driver_test.go @@ -1448,7 +1448,7 @@ func TestCollation(t *testing.T) { t.Skipf("MySQL server not running on %s", netAddr) } - defaultCollation := "utf8_general_ci" + defaultCollation := "utf8mb4_general_ci" testCollations := []string{ "", // do not set defaultCollation, // driver default diff --git a/dsn_test.go b/dsn_test.go index 280fdf61f..fe1572dea 100644 --- a/dsn_test.go +++ b/dsn_test.go @@ -22,55 +22,55 @@ var testDSNs = []struct { out *Config }{{ "username:password@protocol(address)/dbname?param=value", - &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, }, { "username:password@protocol(address)/dbname?param=value&columnsWithAlias=true", - &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, ColumnsWithAlias: true}, + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, ColumnsWithAlias: true}, }, { "username:password@protocol(address)/dbname?param=value&columnsWithAlias=true&multiStatements=true", - &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, ColumnsWithAlias: true, MultiStatements: true}, + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, ColumnsWithAlias: true, MultiStatements: true}, }, { "user@unix(/path/to/socket)/dbname?charset=utf8", - &Config{User: "user", Net: "unix", Addr: "/path/to/socket", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{User: "user", Net: "unix", Addr: "/path/to/socket", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, }, { "user:password@tcp(localhost:5555)/dbname?charset=utf8&tls=true", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, TLSConfig: "true"}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, TLSConfig: "true"}, }, { "user:password@tcp(localhost:5555)/dbname?charset=utf8mb4,utf8&tls=skip-verify", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8mb4,utf8"}, Collation: "utf8_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, TLSConfig: "skip-verify"}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8mb4,utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, TLSConfig: "skip-verify"}, }, { "user:password@/dbname?loc=UTC&timeout=30s&readTimeout=1s&writeTimeout=1s&allowAllFiles=1&clientFoundRows=true&allowOldPasswords=TRUE&collation=utf8mb4_unicode_ci&maxAllowedPacket=16777216", &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_unicode_ci", Loc: time.UTC, AllowNativePasswords: true, Timeout: 30 * time.Second, ReadTimeout: time.Second, WriteTimeout: time.Second, AllowAllFiles: true, AllowOldPasswords: true, ClientFoundRows: true, MaxAllowedPacket: 16777216}, }, { "user:password@/dbname?allowNativePasswords=false&maxAllowedPacket=0", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, AllowNativePasswords: false}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, AllowNativePasswords: false}, }, { "user:p@ss(word)@tcp([de:ad:be:ef::ca:fe]:80)/dbname?loc=Local", - &Config{User: "user", Passwd: "p@ss(word)", Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:80", DBName: "dbname", Collation: "utf8_general_ci", Loc: time.Local, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{User: "user", Passwd: "p@ss(word)", Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:80", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.Local, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, }, { "/dbname", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, }, { "@/", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, }, { "/", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, }, { "", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, }, { "user:p@/ssword@/", - &Config{User: "user", Passwd: "p@/ssword", Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{User: "user", Passwd: "p@/ssword", Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, }, { "unix/?arg=%2Fsome%2Fpath.ext", - &Config{Net: "unix", Addr: "/tmp/mysql.sock", Params: map[string]string{"arg": "/some/path.ext"}, Collation: "utf8_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "unix", Addr: "/tmp/mysql.sock", Params: map[string]string{"arg": "/some/path.ext"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, }, { "tcp(127.0.0.1)/dbname", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, }, { "tcp(de:ad:be:ef::ca:fe)/dbname", - &Config{Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:3306", DBName: "dbname", Collation: "utf8_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, }, } From 578c4c8066964679ef44f45de2b6c7e811cc665e Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 10 Apr 2019 12:04:54 +0900 Subject: [PATCH 25/93] fix appengine (#950) --- conncheck.go | 2 +- conncheck_windows.go => conncheck_dummy.go | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) rename conncheck_windows.go => conncheck_dummy.go (93%) diff --git a/conncheck.go b/conncheck.go index fa868e84d..cc47aa559 100644 --- a/conncheck.go +++ b/conncheck.go @@ -6,7 +6,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. -// +build !windows +// +build !windows,!appengine package mysql diff --git a/conncheck_windows.go b/conncheck_dummy.go similarity index 93% rename from conncheck_windows.go rename to conncheck_dummy.go index 3d9e63f66..fd01f64c9 100644 --- a/conncheck_windows.go +++ b/conncheck_dummy.go @@ -1,5 +1,3 @@ -package mysql - // Go MySQL Driver - A MySQL-Driver for Go's database/sql package // // Copyright 2019 The Go-MySQL-Driver Authors. All rights reserved. @@ -8,6 +6,10 @@ package mysql // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. +// +build windows appengine + +package mysql + import "net" func connCheck(c net.Conn) error { From d0a548181995c293eb09c61ef80099ba1cdbe8f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicent=20Mart=C3=AD?= Date: Tue, 23 Apr 2019 13:20:50 +0200 Subject: [PATCH 26/93] connection: interpolate uint64 parameters (#955) PR #838 introduced a fix for the driver's custom Value Converter that stopped emitting large uint64 `driver.Value`s as a string. Instead, now _all_ uint{8,16,32,64} values passed to the driver are returned as uint64, and `packets.c` now explicitly handles `driver.Value` instances that are uint64. However, the update in `packets.c` only applies when sending `driver.Value` arguments to the server. When a connection is set up using `InterpolateParams = true` and query interpolation happens inside of the driver, the `(*mysqlConn) interpolateParams` does **not** handle uint64 values (which, again, are now passed by `database/sql` because we've updated our value converter to generate them). Because of this, any `DB.Query` operations which have an uint argument (regardless of its size!!) will force the driver to return `driver.ErrSkip`, disabling client interpolation for such queries. We can fix this by updating `interpolateParams` like we previously updated `writeExecutePacket`. --- connection.go | 3 +++ connection_test.go | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/connection.go b/connection.go index 265fd4e47..565a5480a 100644 --- a/connection.go +++ b/connection.go @@ -213,6 +213,9 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin switch v := arg.(type) { case int64: buf = strconv.AppendInt(buf, v, 10) + case uint64: + // Handle uint64 explicitly because our custom ConvertValue emits unsigned values + buf = strconv.AppendUint(buf, v, 10) case float64: buf = strconv.AppendFloat(buf, v, 'g', -1, 64) case bool: diff --git a/connection_test.go b/connection_test.go index 8e78f36c6..19c17ff8b 100644 --- a/connection_test.go +++ b/connection_test.go @@ -69,6 +69,24 @@ func TestInterpolateParamsPlaceholderInString(t *testing.T) { } } +func TestInterpolateParamsUint64(t *testing.T) { + mc := &mysqlConn{ + buf: newBuffer(nil), + maxAllowedPacket: maxPacketSize, + cfg: &Config{ + InterpolateParams: true, + }, + } + + q, err := mc.interpolateParams("SELECT ?", []driver.Value{uint64(42)}) + if err != nil { + t.Errorf("Expected err=nil, got err=%#v, q=%#v", err, q) + } + if q != "SELECT 42" { + t.Errorf("Expected uint64 interpolation to work, got q=%#v", q) + } +} + func TestCheckNamedValue(t *testing.T) { value := driver.NamedValue{Value: ^uint64(0)} x := &mysqlConn{} From 8056f2ca4aa7be2e2e10ab01426a630d5a5bfa81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicent=20Mart=C3=AD?= Date: Tue, 7 May 2019 11:00:41 +0200 Subject: [PATCH 27/93] packets: reset read deadline before conn check (#964) * packets: reset read deadline before conn check If a MySQL connection has been configured with a short `ReadTimeout`, each read from the TCP connection will be preceded by a `SetReadDeadline` call, which lingers until the next `SetReadDeadline`. This can be an issue if the connection becomes stale after staying too long in the connection pool, because when we attempt to perform a stale connection check, the Go runtime scheduler will return a timedout error from the scheduler itself, without letting us get to the kernel to perform the non-blocking read. To fix this, reset the read deadline before we perform the connection check. * packets: set a 0 deadline --- packets.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packets.go b/packets.go index cbed325f4..30b3352c2 100644 --- a/packets.go +++ b/packets.go @@ -108,7 +108,17 @@ func (mc *mysqlConn) writePacket(data []byte) error { if mc.rawConn != nil { conn = mc.rawConn } - if err := connCheck(conn); err != nil { + var err error + // If this connection has a ReadTimeout which we've been setting on + // reads, reset it to its default value before we attempt a non-blocking + // read, otherwise the scheduler will just time us out before we can read + if mc.cfg.ReadTimeout != 0 { + err = conn.SetReadDeadline(time.Time{}) + } + if err == nil { + err = connCheck(conn) + } + if err != nil { errLog.Print("closing bad idle connection: ", err) mc.Close() return driver.ErrBadConn From 877a9775f06853f611fb2d4e817d92479242d1cd Mon Sep 17 00:00:00 2001 From: Brandon Bennett Date: Fri, 10 May 2019 04:23:35 -0600 Subject: [PATCH 28/93] move tls and pubkey object creation to Config.normalize() (#958) This is still less than ideal since we cannot directly pass in tls.Config into Config and have it be used, but it is sill backwards compatable. In the future this should be revisited to be able to use a custome tls.Config passed directly in without string parsing/registering. --- AUTHORS | 3 ++- dsn.go | 50 +++++++++++++++++++++++++++----------------------- dsn_test.go | 48 ++++++++++++++++++++++++++++++++++++++++++++++-- utils.go | 2 +- 4 files changed, 76 insertions(+), 27 deletions(-) diff --git a/AUTHORS b/AUTHORS index cdf7ae3b8..bfe74c4e1 100644 --- a/AUTHORS +++ b/AUTHORS @@ -90,11 +90,12 @@ Zhenye Xie Barracuda Networks, Inc. Counting Ltd. +Facebook Inc. GitHub Inc. Google Inc. InfoSum Ltd. Keybase Inc. +Multiplay Ltd. Percona LLC Pivotal Inc. Stripe Inc. -Multiplay Ltd. diff --git a/dsn.go b/dsn.go index 6e19ab717..1d9b4ab0a 100644 --- a/dsn.go +++ b/dsn.go @@ -113,17 +113,35 @@ func (cfg *Config) normalize() error { default: return errors.New("default addr for network '" + cfg.Net + "' unknown") } - } else if cfg.Net == "tcp" { cfg.Addr = ensureHavePort(cfg.Addr) } - if cfg.tls != nil { - if cfg.tls.ServerName == "" && !cfg.tls.InsecureSkipVerify { - host, _, err := net.SplitHostPort(cfg.Addr) - if err == nil { - cfg.tls.ServerName = host - } + switch cfg.TLSConfig { + case "false", "": + // don't set anything + case "true": + cfg.tls = &tls.Config{} + case "skip-verify", "preferred": + cfg.tls = &tls.Config{InsecureSkipVerify: true} + default: + cfg.tls = getTLSConfigClone(cfg.TLSConfig) + if cfg.tls == nil { + return errors.New("invalid value / unknown config name: " + cfg.TLSConfig) + } + } + + if cfg.tls != nil && cfg.tls.ServerName == "" && !cfg.tls.InsecureSkipVerify { + host, _, err := net.SplitHostPort(cfg.Addr) + if err == nil { + cfg.tls.ServerName = host + } + } + + if cfg.ServerPubKey != "" { + cfg.pubKey = getServerPubKey(cfg.ServerPubKey) + if cfg.pubKey == nil { + return errors.New("invalid value / unknown server pub key name: " + cfg.ServerPubKey) } } @@ -552,13 +570,7 @@ func parseDSNParams(cfg *Config, params string) (err error) { if err != nil { return fmt.Errorf("invalid value for server pub key name: %v", err) } - - if pubKey := getServerPubKey(name); pubKey != nil { - cfg.ServerPubKey = name - cfg.pubKey = pubKey - } else { - return errors.New("invalid value / unknown server pub key name: " + name) - } + cfg.ServerPubKey = name // Strict mode case "strict": @@ -577,25 +589,17 @@ func parseDSNParams(cfg *Config, params string) (err error) { if isBool { if boolValue { cfg.TLSConfig = "true" - cfg.tls = &tls.Config{} } else { cfg.TLSConfig = "false" } } else if vl := strings.ToLower(value); vl == "skip-verify" || vl == "preferred" { cfg.TLSConfig = vl - cfg.tls = &tls.Config{InsecureSkipVerify: true} } else { name, err := url.QueryUnescape(value) if err != nil { return fmt.Errorf("invalid value for TLS config name: %v", err) } - - if tlsConfig := getTLSConfigClone(name); tlsConfig != nil { - cfg.TLSConfig = name - cfg.tls = tlsConfig - } else { - return errors.New("invalid value / unknown config name: " + name) - } + cfg.TLSConfig = name } // I/O write Timeout diff --git a/dsn_test.go b/dsn_test.go index fe1572dea..50dc2932c 100644 --- a/dsn_test.go +++ b/dsn_test.go @@ -39,8 +39,8 @@ var testDSNs = []struct { "user:password@tcp(localhost:5555)/dbname?charset=utf8mb4,utf8&tls=skip-verify", &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8mb4,utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, TLSConfig: "skip-verify"}, }, { - "user:password@/dbname?loc=UTC&timeout=30s&readTimeout=1s&writeTimeout=1s&allowAllFiles=1&clientFoundRows=true&allowOldPasswords=TRUE&collation=utf8mb4_unicode_ci&maxAllowedPacket=16777216", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_unicode_ci", Loc: time.UTC, AllowNativePasswords: true, Timeout: 30 * time.Second, ReadTimeout: time.Second, WriteTimeout: time.Second, AllowAllFiles: true, AllowOldPasswords: true, ClientFoundRows: true, MaxAllowedPacket: 16777216}, + "user:password@/dbname?loc=UTC&timeout=30s&readTimeout=1s&writeTimeout=1s&allowAllFiles=1&clientFoundRows=true&allowOldPasswords=TRUE&collation=utf8mb4_unicode_ci&maxAllowedPacket=16777216&tls=false&allowCleartextPasswords=true&parseTime=true&rejectReadOnly=true", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_unicode_ci", Loc: time.UTC, TLSConfig: "false", AllowCleartextPasswords: true, AllowNativePasswords: true, Timeout: 30 * time.Second, ReadTimeout: time.Second, WriteTimeout: time.Second, AllowAllFiles: true, AllowOldPasswords: true, ClientFoundRows: true, MaxAllowedPacket: 16777216, ParseTime: true, RejectReadOnly: true}, }, { "user:password@/dbname?allowNativePasswords=false&maxAllowedPacket=0", &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, AllowNativePasswords: false}, @@ -358,6 +358,50 @@ func TestCloneConfig(t *testing.T) { } } +func TestNormalizeTLSConfig(t *testing.T) { + tt := []struct { + tlsConfig string + want *tls.Config + }{ + {"", nil}, + {"false", nil}, + {"true", &tls.Config{ServerName: "myserver"}}, + {"skip-verify", &tls.Config{InsecureSkipVerify: true}}, + {"preferred", &tls.Config{InsecureSkipVerify: true}}, + {"test_tls_config", &tls.Config{ServerName: "myServerName"}}, + } + + RegisterTLSConfig("test_tls_config", &tls.Config{ServerName: "myServerName"}) + defer func() { DeregisterTLSConfig("test_tls_config") }() + + for _, tc := range tt { + t.Run(tc.tlsConfig, func(t *testing.T) { + cfg := &Config{ + Addr: "myserver:3306", + TLSConfig: tc.tlsConfig, + } + + cfg.normalize() + + if cfg.tls == nil { + if tc.want != nil { + t.Fatal("wanted a tls config but got nil instead") + } + return + } + + if cfg.tls.ServerName != tc.want.ServerName { + t.Errorf("tls.ServerName doesn't match (want: '%s', got: '%s')", + tc.want.ServerName, cfg.tls.ServerName) + } + if cfg.tls.InsecureSkipVerify != tc.want.InsecureSkipVerify { + t.Errorf("tls.InsecureSkipVerify doesn't match (want: %T, got :%T)", + tc.want.InsecureSkipVerify, cfg.tls.InsecureSkipVerify) + } + }) + } +} + func BenchmarkParseDSN(b *testing.B) { b.ReportAllocs() diff --git a/utils.go b/utils.go index 201691fe8..cfa10e9c0 100644 --- a/utils.go +++ b/utils.go @@ -56,7 +56,7 @@ var ( // db, err := sql.Open("mysql", "user@tcp(localhost:3306)/test?tls=custom") // func RegisterTLSConfig(key string, config *tls.Config) error { - if _, isBool := readBool(key); isBool || strings.ToLower(key) == "skip-verify" { + if _, isBool := readBool(key); isBool || strings.ToLower(key) == "skip-verify" || strings.ToLower(key) == "preferred" { return fmt.Errorf("key '%s' is reserved", key) } From 7244e501438d2bcefc1474a6b2fa101200ffee82 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 4 Sep 2019 22:05:19 +0900 Subject: [PATCH 29/93] try fix osx build on Travis (#994) --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index eae311b14..9d3139207 100644 --- a/.travis.yml +++ b/.travis.yml @@ -105,6 +105,7 @@ matrix: homebrew: packages: - mysql + update: true go: 1.12.x before_install: - go get golang.org/x/tools/cmd/cover From 23821f42b6688625ba2202899226983dc7055fad Mon Sep 17 00:00:00 2001 From: Vladimir Kovpak Date: Wed, 4 Sep 2019 17:58:47 +0300 Subject: [PATCH 30/93] GCP DSN example for flexible environment (#993) * Added GCP connection string example for flexible environment. * Added one more GCP connection string example for flexible environment. * Unified GCP connection strings examples for 2nd gen and flexible env. --- AUTHORS | 1 + README.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index bfe74c4e1..d87414e82 100644 --- a/AUTHORS +++ b/AUTHORS @@ -81,6 +81,7 @@ Steven Hartland Thomas Wodarek Tim Ruffles Tom Jenkinson +Vladimir Kovpak Xiangyu Hu Xiaobing Jiang Xiuming Chen diff --git a/README.md b/README.md index c6adf1d63..20fff486f 100644 --- a/README.md +++ b/README.md @@ -403,7 +403,7 @@ user@cloudsql(project-id:instance-name)/dbname Google Cloud SQL on App Engine (Second Generation MySQL Server): ``` -user@cloudsql(project-id:regionname:instance-name)/dbname +user:password@unix(/cloudsql/project-id:region-name:instance-name)/dbname ``` TCP using default port (3306) on localhost: From d9aa6d3abe0f1072adb4aa71e1eb1d3eb3d3921c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Mengu=C3=A9?= Date: Thu, 5 Sep 2019 05:54:07 +0200 Subject: [PATCH 31/93] Refactor NullTime as go1.13's sql.NullTime (#995) --- nulltime.go | 50 +++++++++++++++++++++++++++++++++++++ nulltime_go113.go | 31 +++++++++++++++++++++++ nulltime_legacy.go | 34 +++++++++++++++++++++++++ nulltime_test.go | 62 ++++++++++++++++++++++++++++++++++++++++++++++ utils.go | 54 ---------------------------------------- utils_test.go | 41 ------------------------------ 6 files changed, 177 insertions(+), 95 deletions(-) create mode 100644 nulltime.go create mode 100644 nulltime_go113.go create mode 100644 nulltime_legacy.go create mode 100644 nulltime_test.go diff --git a/nulltime.go b/nulltime.go new file mode 100644 index 000000000..afa8a89e9 --- /dev/null +++ b/nulltime.go @@ -0,0 +1,50 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "database/sql/driver" + "fmt" + "time" +) + +// Scan implements the Scanner interface. +// The value type must be time.Time or string / []byte (formatted time-string), +// otherwise Scan fails. +func (nt *NullTime) Scan(value interface{}) (err error) { + if value == nil { + nt.Time, nt.Valid = time.Time{}, false + return + } + + switch v := value.(type) { + case time.Time: + nt.Time, nt.Valid = v, true + return + case []byte: + nt.Time, err = parseDateTime(string(v), time.UTC) + nt.Valid = (err == nil) + return + case string: + nt.Time, err = parseDateTime(v, time.UTC) + nt.Valid = (err == nil) + return + } + + nt.Valid = false + return fmt.Errorf("Can't convert %T to time.Time", value) +} + +// Value implements the driver Valuer interface. +func (nt NullTime) Value() (driver.Value, error) { + if !nt.Valid { + return nil, nil + } + return nt.Time, nil +} diff --git a/nulltime_go113.go b/nulltime_go113.go new file mode 100644 index 000000000..c392594dd --- /dev/null +++ b/nulltime_go113.go @@ -0,0 +1,31 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build go1.13 + +package mysql + +import ( + "database/sql" +) + +// NullTime represents a time.Time that may be NULL. +// NullTime implements the Scanner interface so +// it can be used as a scan destination: +// +// var nt NullTime +// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) +// ... +// if nt.Valid { +// // use nt.Time +// } else { +// // NULL value +// } +// +// This NullTime implementation is not driver-specific +type NullTime sql.NullTime diff --git a/nulltime_legacy.go b/nulltime_legacy.go new file mode 100644 index 000000000..86d159d44 --- /dev/null +++ b/nulltime_legacy.go @@ -0,0 +1,34 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build !go1.13 + +package mysql + +import ( + "time" +) + +// NullTime represents a time.Time that may be NULL. +// NullTime implements the Scanner interface so +// it can be used as a scan destination: +// +// var nt NullTime +// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) +// ... +// if nt.Valid { +// // use nt.Time +// } else { +// // NULL value +// } +// +// This NullTime implementation is not driver-specific +type NullTime struct { + Time time.Time + Valid bool // Valid is true if Time is not NULL +} diff --git a/nulltime_test.go b/nulltime_test.go new file mode 100644 index 000000000..a14ec0607 --- /dev/null +++ b/nulltime_test.go @@ -0,0 +1,62 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "database/sql" + "database/sql/driver" + "testing" + "time" +) + +var ( + // Check implementation of interfaces + _ driver.Valuer = NullTime{} + _ sql.Scanner = (*NullTime)(nil) +) + +func TestScanNullTime(t *testing.T) { + var scanTests = []struct { + in interface{} + error bool + valid bool + time time.Time + }{ + {tDate, false, true, tDate}, + {sDate, false, true, tDate}, + {[]byte(sDate), false, true, tDate}, + {tDateTime, false, true, tDateTime}, + {sDateTime, false, true, tDateTime}, + {[]byte(sDateTime), false, true, tDateTime}, + {tDate0, false, true, tDate0}, + {sDate0, false, true, tDate0}, + {[]byte(sDate0), false, true, tDate0}, + {sDateTime0, false, true, tDate0}, + {[]byte(sDateTime0), false, true, tDate0}, + {"", true, false, tDate0}, + {"1234", true, false, tDate0}, + {0, true, false, tDate0}, + } + + var nt = NullTime{} + var err error + + for _, tst := range scanTests { + err = nt.Scan(tst.in) + if (err != nil) != tst.error { + t.Errorf("%v: expected error status %t, got %t", tst.in, tst.error, (err != nil)) + } + if nt.Valid != tst.valid { + t.Errorf("%v: expected valid status %t, got %t", tst.in, tst.valid, nt.Valid) + } + if nt.Time != tst.time { + t.Errorf("%v: expected time %v, got %v", tst.in, tst.time, nt.Time) + } + } +} diff --git a/utils.go b/utils.go index cfa10e9c0..9552e80b5 100644 --- a/utils.go +++ b/utils.go @@ -106,60 +106,6 @@ func readBool(input string) (value bool, valid bool) { * Time related utils * ******************************************************************************/ -// NullTime represents a time.Time that may be NULL. -// NullTime implements the Scanner interface so -// it can be used as a scan destination: -// -// var nt NullTime -// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) -// ... -// if nt.Valid { -// // use nt.Time -// } else { -// // NULL value -// } -// -// This NullTime implementation is not driver-specific -type NullTime struct { - Time time.Time - Valid bool // Valid is true if Time is not NULL -} - -// Scan implements the Scanner interface. -// The value type must be time.Time or string / []byte (formatted time-string), -// otherwise Scan fails. -func (nt *NullTime) Scan(value interface{}) (err error) { - if value == nil { - nt.Time, nt.Valid = time.Time{}, false - return - } - - switch v := value.(type) { - case time.Time: - nt.Time, nt.Valid = v, true - return - case []byte: - nt.Time, err = parseDateTime(string(v), time.UTC) - nt.Valid = (err == nil) - return - case string: - nt.Time, err = parseDateTime(v, time.UTC) - nt.Valid = (err == nil) - return - } - - nt.Valid = false - return fmt.Errorf("Can't convert %T to time.Time", value) -} - -// Value implements the driver Valuer interface. -func (nt NullTime) Value() (driver.Value, error) { - if !nt.Valid { - return nil, nil - } - return nt.Time, nil -} - func parseDateTime(str string, loc *time.Location) (t time.Time, err error) { base := "0000-00-00 00:00:00.0000000" switch len(str) { diff --git a/utils_test.go b/utils_test.go index 8951a7a81..10a60c2d0 100644 --- a/utils_test.go +++ b/utils_test.go @@ -14,49 +14,8 @@ import ( "database/sql/driver" "encoding/binary" "testing" - "time" ) -func TestScanNullTime(t *testing.T) { - var scanTests = []struct { - in interface{} - error bool - valid bool - time time.Time - }{ - {tDate, false, true, tDate}, - {sDate, false, true, tDate}, - {[]byte(sDate), false, true, tDate}, - {tDateTime, false, true, tDateTime}, - {sDateTime, false, true, tDateTime}, - {[]byte(sDateTime), false, true, tDateTime}, - {tDate0, false, true, tDate0}, - {sDate0, false, true, tDate0}, - {[]byte(sDate0), false, true, tDate0}, - {sDateTime0, false, true, tDate0}, - {[]byte(sDateTime0), false, true, tDate0}, - {"", true, false, tDate0}, - {"1234", true, false, tDate0}, - {0, true, false, tDate0}, - } - - var nt = NullTime{} - var err error - - for _, tst := range scanTests { - err = nt.Scan(tst.in) - if (err != nil) != tst.error { - t.Errorf("%v: expected error status %t, got %t", tst.in, tst.error, (err != nil)) - } - if nt.Valid != tst.valid { - t.Errorf("%v: expected valid status %t, got %t", tst.in, tst.valid, nt.Valid) - } - if nt.Time != tst.time { - t.Errorf("%v: expected time %v, got %v", tst.in, tst.time, nt.Time) - } - } -} - func TestLengthEncodedInteger(t *testing.T) { var integerTests = []struct { num uint64 From b2c03bcae3d4bf9658e63ad4b9b32581d810fb20 Mon Sep 17 00:00:00 2001 From: Julien Schmidt Date: Sat, 7 Sep 2019 14:21:37 +0200 Subject: [PATCH 32/93] conncheck: move var declarations into closure (#997) --- conncheck.go | 43 ++++++++++++++++++++++--------------------- conncheck_dummy.go | 2 +- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/conncheck.go b/conncheck.go index cc47aa559..70e9925f6 100644 --- a/conncheck.go +++ b/conncheck.go @@ -19,35 +19,36 @@ import ( var errUnexpectedRead = errors.New("unexpected read from socket") -func connCheck(c net.Conn) error { - var ( - n int - err error - buff [1]byte - ) - - sconn, ok := c.(syscall.Conn) +func connCheck(conn net.Conn) error { + var sysErr error + + sysConn, ok := conn.(syscall.Conn) if !ok { return nil } - rc, err := sconn.SyscallConn() + rawConn, err := sysConn.SyscallConn() if err != nil { return err } - rerr := rc.Read(func(fd uintptr) bool { - n, err = syscall.Read(int(fd), buff[:]) + + err = rawConn.Read(func(fd uintptr) bool { + var buf [1]byte + n, err := syscall.Read(int(fd), buf[:]) + switch { + case n == 0 && err == nil: + sysErr = io.EOF + case n > 0: + sysErr = errUnexpectedRead + case err == syscall.EAGAIN || err == syscall.EWOULDBLOCK: + sysErr = nil + default: + sysErr = err + } return true }) - switch { - case rerr != nil: - return rerr - case n == 0 && err == nil: - return io.EOF - case n > 0: - return errUnexpectedRead - case err == syscall.EAGAIN || err == syscall.EWOULDBLOCK: - return nil - default: + if err != nil { return err } + + return sysErr } diff --git a/conncheck_dummy.go b/conncheck_dummy.go index fd01f64c9..4888288aa 100644 --- a/conncheck_dummy.go +++ b/conncheck_dummy.go @@ -12,6 +12,6 @@ package mysql import "net" -func connCheck(c net.Conn) error { +func connCheck(conn net.Conn) error { return nil } From 6c79a37b2e06d0297e8a76a70b747f72bdfa8338 Mon Sep 17 00:00:00 2001 From: Julien Schmidt Date: Tue, 17 Sep 2019 11:31:45 +0200 Subject: [PATCH 33/93] move CONTRIBUTING.md to .github/CONTRIBUTING.md (#1000) --- CONTRIBUTING.md => .github/CONTRIBUTING.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename CONTRIBUTING.md => .github/CONTRIBUTING.md (100%) diff --git a/CONTRIBUTING.md b/.github/CONTRIBUTING.md similarity index 100% rename from CONTRIBUTING.md rename to .github/CONTRIBUTING.md From 59de189eea30b4ff6207a44f810232907f95b098 Mon Sep 17 00:00:00 2001 From: Julien Schmidt Date: Tue, 1 Oct 2019 08:08:38 +0200 Subject: [PATCH 34/93] Remove Cloud SQL dialer (#1007) CloudSQL is only available up to Go 1.9 on Google AppEngine, which was phased out. Starting from 2019-10-01, no new apps can be deployed to GAE/Go 1.9 anymore. This dialer is thus obsolete. Fixes #1002 --- README.md | 7 +------ appengine.go | 25 ------------------------- 2 files changed, 1 insertion(+), 31 deletions(-) delete mode 100644 appengine.go diff --git a/README.md b/README.md index 20fff486f..dce7a5325 100644 --- a/README.md +++ b/README.md @@ -396,12 +396,7 @@ TCP on a remote host, e.g. Amazon RDS: id:password@tcp(your-amazonaws-uri.com:3306)/dbname ``` -Google Cloud SQL on App Engine (First Generation MySQL Server): -``` -user@cloudsql(project-id:instance-name)/dbname -``` - -Google Cloud SQL on App Engine (Second Generation MySQL Server): +Google Cloud SQL on App Engine: ``` user:password@unix(/cloudsql/project-id:region-name:instance-name)/dbname ``` diff --git a/appengine.go b/appengine.go deleted file mode 100644 index 914e6623b..000000000 --- a/appengine.go +++ /dev/null @@ -1,25 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -// +build appengine - -package mysql - -import ( - "context" - "net" - - "google.golang.org/appengine/cloudsql" -) - -func init() { - RegisterDialContext("cloudsql", func(_ context.Context, instance string) (net.Conn, error) { - // XXX: the cloudsql driver still does not export a Context-aware dialer. - return cloudsql.Dial(instance) - }) -} From 14bb9c0fc20fc57f6c3339adeac7f469768d9524 Mon Sep 17 00:00:00 2001 From: Julien Schmidt Date: Tue, 1 Oct 2019 08:09:45 +0200 Subject: [PATCH 35/93] Add go.mod (#1003) --- go.mod | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 go.mod diff --git a/go.mod b/go.mod new file mode 100644 index 000000000..29e5d800d --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/go-sql-driver/mysql + +go 1.9 From 5ee934fb0eb6903ba5396bf34817ff178b29de39 Mon Sep 17 00:00:00 2001 From: Johann Forster Date: Mon, 21 Oct 2019 12:25:53 +0200 Subject: [PATCH 36/93] Add string type to ColumnTypeScanType(). (#976) ColumnTypeScanType() now returns string and sql.NullString. --- AUTHORS | 1 + driver_test.go | 15 +++++++++------ fields.go | 52 +++++++++++++++++++++++++++++++------------------- 3 files changed, 42 insertions(+), 26 deletions(-) diff --git a/AUTHORS b/AUTHORS index d87414e82..dd27253e3 100644 --- a/AUTHORS +++ b/AUTHORS @@ -45,6 +45,7 @@ Jeff Hodges Jeffrey Charles Jerome Meyer Jian Zhen +Johann Forster Joshua Prunier Julien Lefevre Julien Schmidt diff --git a/driver_test.go b/driver_test.go index 3dee1bab2..e6c7f83a0 100644 --- a/driver_test.go +++ b/driver_test.go @@ -2733,6 +2733,9 @@ func TestRowsColumnTypes(t *testing.T) { rb0pad4 := sql.RawBytes("0\x00\x00\x00") // BINARY right-pads values with 0x00 rbx0 := sql.RawBytes("\x00") rbx42 := sql.RawBytes("\x42") + s0 := sql.NullString{String: "0", Valid: true} + sNULL := sql.NullString{String: "", Valid: false} + sTest := sql.NullString{String: "Test", Valid: true} var columns = []struct { name string @@ -2771,18 +2774,18 @@ func TestRowsColumnTypes(t *testing.T) { {"decimal2null", "DECIMAL(8,4)", "DECIMAL", scanTypeRawBytes, true, 8, 4, [3]string{"0", "NULL", "1234.123456"}, [3]interface{}{sql.RawBytes("0.0000"), rbNULL, sql.RawBytes("1234.1235")}}, {"decimal3", "DECIMAL(5,0) NOT NULL", "DECIMAL", scanTypeRawBytes, false, 5, 0, [3]string{"0", "13.37", "-12345.123456"}, [3]interface{}{rb0, sql.RawBytes("13"), sql.RawBytes("-12345")}}, {"decimal3null", "DECIMAL(5,0)", "DECIMAL", scanTypeRawBytes, true, 5, 0, [3]string{"0", "NULL", "-12345.123456"}, [3]interface{}{rb0, rbNULL, sql.RawBytes("-12345")}}, - {"char25null", "CHAR(25)", "CHAR", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, - {"varchar42", "VARCHAR(42) NOT NULL", "VARCHAR", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"char25null", "CHAR(25)", "CHAR", scanTypeNullString, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{s0, sNULL, sTest}}, + {"varchar42", "VARCHAR(42) NOT NULL", "VARCHAR", scanTypeString, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{"0", "Test", "42"}}, {"binary4null", "BINARY(4)", "BINARY", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0pad4, rbNULL, rbTest}}, {"varbinary42", "VARBINARY(42) NOT NULL", "VARBINARY", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, {"tinyblobnull", "TINYBLOB", "BLOB", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, - {"tinytextnull", "TINYTEXT", "TEXT", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, + {"tinytextnull", "TINYTEXT", "TEXT", scanTypeNullString, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{s0, sNULL, sTest}}, {"blobnull", "BLOB", "BLOB", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, - {"textnull", "TEXT", "TEXT", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, + {"textnull", "TEXT", "TEXT", scanTypeNullString, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{s0, sNULL, sTest}}, {"mediumblob", "MEDIUMBLOB NOT NULL", "BLOB", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, - {"mediumtext", "MEDIUMTEXT NOT NULL", "TEXT", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"mediumtext", "MEDIUMTEXT NOT NULL", "TEXT", scanTypeString, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{"0", "Test", "42"}}, {"longblob", "LONGBLOB NOT NULL", "BLOB", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, - {"longtext", "LONGTEXT NOT NULL", "TEXT", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"longtext", "LONGTEXT NOT NULL", "TEXT", scanTypeString, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{"0", "Test", "42"}}, {"datetime", "DATETIME", "DATETIME", scanTypeNullTime, true, 0, 0, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt0, nt0}}, {"datetime2", "DATETIME(2)", "DATETIME", scanTypeNullTime, true, 2, 2, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt1, nt2}}, {"datetime6", "DATETIME(6)", "DATETIME", scanTypeNullTime, true, 6, 6, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt1, nt6}}, diff --git a/fields.go b/fields.go index e1e2ece4b..ad2207cb3 100644 --- a/fields.go +++ b/fields.go @@ -98,21 +98,23 @@ func (mf *mysqlField) typeDatabaseName() string { } var ( - scanTypeFloat32 = reflect.TypeOf(float32(0)) - scanTypeFloat64 = reflect.TypeOf(float64(0)) - scanTypeInt8 = reflect.TypeOf(int8(0)) - scanTypeInt16 = reflect.TypeOf(int16(0)) - scanTypeInt32 = reflect.TypeOf(int32(0)) - scanTypeInt64 = reflect.TypeOf(int64(0)) - scanTypeNullFloat = reflect.TypeOf(sql.NullFloat64{}) - scanTypeNullInt = reflect.TypeOf(sql.NullInt64{}) - scanTypeNullTime = reflect.TypeOf(NullTime{}) - scanTypeUint8 = reflect.TypeOf(uint8(0)) - scanTypeUint16 = reflect.TypeOf(uint16(0)) - scanTypeUint32 = reflect.TypeOf(uint32(0)) - scanTypeUint64 = reflect.TypeOf(uint64(0)) - scanTypeRawBytes = reflect.TypeOf(sql.RawBytes{}) - scanTypeUnknown = reflect.TypeOf(new(interface{})) + scanTypeFloat32 = reflect.TypeOf(float32(0)) + scanTypeFloat64 = reflect.TypeOf(float64(0)) + scanTypeInt8 = reflect.TypeOf(int8(0)) + scanTypeInt16 = reflect.TypeOf(int16(0)) + scanTypeInt32 = reflect.TypeOf(int32(0)) + scanTypeInt64 = reflect.TypeOf(int64(0)) + scanTypeNullFloat = reflect.TypeOf(sql.NullFloat64{}) + scanTypeNullInt = reflect.TypeOf(sql.NullInt64{}) + scanTypeNullTime = reflect.TypeOf(NullTime{}) + scanTypeUint8 = reflect.TypeOf(uint8(0)) + scanTypeUint16 = reflect.TypeOf(uint16(0)) + scanTypeUint32 = reflect.TypeOf(uint32(0)) + scanTypeUint64 = reflect.TypeOf(uint64(0)) + scanTypeString = reflect.TypeOf(string("")) + scanTypeNullString = reflect.TypeOf(sql.NullString{}) + scanTypeRawBytes = reflect.TypeOf(sql.RawBytes{}) + scanTypeUnknown = reflect.TypeOf(new(interface{})) ) type mysqlField struct { @@ -175,11 +177,21 @@ func (mf *mysqlField) scanType() reflect.Type { } return scanTypeNullFloat - case fieldTypeDecimal, fieldTypeNewDecimal, fieldTypeVarChar, - fieldTypeBit, fieldTypeEnum, fieldTypeSet, fieldTypeTinyBLOB, - fieldTypeMediumBLOB, fieldTypeLongBLOB, fieldTypeBLOB, - fieldTypeVarString, fieldTypeString, fieldTypeGeometry, fieldTypeJSON, - fieldTypeTime: + case fieldTypeVarChar, fieldTypeString, fieldTypeVarString, fieldTypeTinyBLOB, + fieldTypeMediumBLOB, fieldTypeLongBLOB, fieldTypeBLOB: + // charsetnr == 63 means this column is binary. + // https://dev.mysql.com/doc/refman/8.0/en/c-api-data-structures.html + if mf.charSet == 63 { + return scanTypeRawBytes + } + if mf.flags&flagNotNULL != 0 { + return scanTypeString + } + return scanTypeNullString + + case fieldTypeDecimal, fieldTypeNewDecimal, fieldTypeBit, + fieldTypeEnum, fieldTypeSet, + fieldTypeGeometry, fieldTypeJSON, fieldTypeTime: return scanTypeRawBytes case fieldTypeDate, fieldTypeNewDate, From 6ea7374bc1b0cd6cc6398e2aec912f6764aa5adc Mon Sep 17 00:00:00 2001 From: Julien Schmidt Date: Tue, 22 Oct 2019 13:23:24 +0200 Subject: [PATCH 37/93] Revert "Add string type to ColumnTypeScanType(). (#976)" (#1018) This reverts commit 5ee934fb0eb6903ba5396bf34817ff178b29de39. --- AUTHORS | 1 - driver_test.go | 15 ++++++--------- fields.go | 52 +++++++++++++++++++------------------------------- 3 files changed, 26 insertions(+), 42 deletions(-) diff --git a/AUTHORS b/AUTHORS index dd27253e3..d87414e82 100644 --- a/AUTHORS +++ b/AUTHORS @@ -45,7 +45,6 @@ Jeff Hodges Jeffrey Charles Jerome Meyer Jian Zhen -Johann Forster Joshua Prunier Julien Lefevre Julien Schmidt diff --git a/driver_test.go b/driver_test.go index e6c7f83a0..3dee1bab2 100644 --- a/driver_test.go +++ b/driver_test.go @@ -2733,9 +2733,6 @@ func TestRowsColumnTypes(t *testing.T) { rb0pad4 := sql.RawBytes("0\x00\x00\x00") // BINARY right-pads values with 0x00 rbx0 := sql.RawBytes("\x00") rbx42 := sql.RawBytes("\x42") - s0 := sql.NullString{String: "0", Valid: true} - sNULL := sql.NullString{String: "", Valid: false} - sTest := sql.NullString{String: "Test", Valid: true} var columns = []struct { name string @@ -2774,18 +2771,18 @@ func TestRowsColumnTypes(t *testing.T) { {"decimal2null", "DECIMAL(8,4)", "DECIMAL", scanTypeRawBytes, true, 8, 4, [3]string{"0", "NULL", "1234.123456"}, [3]interface{}{sql.RawBytes("0.0000"), rbNULL, sql.RawBytes("1234.1235")}}, {"decimal3", "DECIMAL(5,0) NOT NULL", "DECIMAL", scanTypeRawBytes, false, 5, 0, [3]string{"0", "13.37", "-12345.123456"}, [3]interface{}{rb0, sql.RawBytes("13"), sql.RawBytes("-12345")}}, {"decimal3null", "DECIMAL(5,0)", "DECIMAL", scanTypeRawBytes, true, 5, 0, [3]string{"0", "NULL", "-12345.123456"}, [3]interface{}{rb0, rbNULL, sql.RawBytes("-12345")}}, - {"char25null", "CHAR(25)", "CHAR", scanTypeNullString, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{s0, sNULL, sTest}}, - {"varchar42", "VARCHAR(42) NOT NULL", "VARCHAR", scanTypeString, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{"0", "Test", "42"}}, + {"char25null", "CHAR(25)", "CHAR", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, + {"varchar42", "VARCHAR(42) NOT NULL", "VARCHAR", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, {"binary4null", "BINARY(4)", "BINARY", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0pad4, rbNULL, rbTest}}, {"varbinary42", "VARBINARY(42) NOT NULL", "VARBINARY", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, {"tinyblobnull", "TINYBLOB", "BLOB", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, - {"tinytextnull", "TINYTEXT", "TEXT", scanTypeNullString, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{s0, sNULL, sTest}}, + {"tinytextnull", "TINYTEXT", "TEXT", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, {"blobnull", "BLOB", "BLOB", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, - {"textnull", "TEXT", "TEXT", scanTypeNullString, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{s0, sNULL, sTest}}, + {"textnull", "TEXT", "TEXT", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, {"mediumblob", "MEDIUMBLOB NOT NULL", "BLOB", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, - {"mediumtext", "MEDIUMTEXT NOT NULL", "TEXT", scanTypeString, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{"0", "Test", "42"}}, + {"mediumtext", "MEDIUMTEXT NOT NULL", "TEXT", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, {"longblob", "LONGBLOB NOT NULL", "BLOB", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, - {"longtext", "LONGTEXT NOT NULL", "TEXT", scanTypeString, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{"0", "Test", "42"}}, + {"longtext", "LONGTEXT NOT NULL", "TEXT", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, {"datetime", "DATETIME", "DATETIME", scanTypeNullTime, true, 0, 0, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt0, nt0}}, {"datetime2", "DATETIME(2)", "DATETIME", scanTypeNullTime, true, 2, 2, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt1, nt2}}, {"datetime6", "DATETIME(6)", "DATETIME", scanTypeNullTime, true, 6, 6, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt1, nt6}}, diff --git a/fields.go b/fields.go index ad2207cb3..e1e2ece4b 100644 --- a/fields.go +++ b/fields.go @@ -98,23 +98,21 @@ func (mf *mysqlField) typeDatabaseName() string { } var ( - scanTypeFloat32 = reflect.TypeOf(float32(0)) - scanTypeFloat64 = reflect.TypeOf(float64(0)) - scanTypeInt8 = reflect.TypeOf(int8(0)) - scanTypeInt16 = reflect.TypeOf(int16(0)) - scanTypeInt32 = reflect.TypeOf(int32(0)) - scanTypeInt64 = reflect.TypeOf(int64(0)) - scanTypeNullFloat = reflect.TypeOf(sql.NullFloat64{}) - scanTypeNullInt = reflect.TypeOf(sql.NullInt64{}) - scanTypeNullTime = reflect.TypeOf(NullTime{}) - scanTypeUint8 = reflect.TypeOf(uint8(0)) - scanTypeUint16 = reflect.TypeOf(uint16(0)) - scanTypeUint32 = reflect.TypeOf(uint32(0)) - scanTypeUint64 = reflect.TypeOf(uint64(0)) - scanTypeString = reflect.TypeOf(string("")) - scanTypeNullString = reflect.TypeOf(sql.NullString{}) - scanTypeRawBytes = reflect.TypeOf(sql.RawBytes{}) - scanTypeUnknown = reflect.TypeOf(new(interface{})) + scanTypeFloat32 = reflect.TypeOf(float32(0)) + scanTypeFloat64 = reflect.TypeOf(float64(0)) + scanTypeInt8 = reflect.TypeOf(int8(0)) + scanTypeInt16 = reflect.TypeOf(int16(0)) + scanTypeInt32 = reflect.TypeOf(int32(0)) + scanTypeInt64 = reflect.TypeOf(int64(0)) + scanTypeNullFloat = reflect.TypeOf(sql.NullFloat64{}) + scanTypeNullInt = reflect.TypeOf(sql.NullInt64{}) + scanTypeNullTime = reflect.TypeOf(NullTime{}) + scanTypeUint8 = reflect.TypeOf(uint8(0)) + scanTypeUint16 = reflect.TypeOf(uint16(0)) + scanTypeUint32 = reflect.TypeOf(uint32(0)) + scanTypeUint64 = reflect.TypeOf(uint64(0)) + scanTypeRawBytes = reflect.TypeOf(sql.RawBytes{}) + scanTypeUnknown = reflect.TypeOf(new(interface{})) ) type mysqlField struct { @@ -177,21 +175,11 @@ func (mf *mysqlField) scanType() reflect.Type { } return scanTypeNullFloat - case fieldTypeVarChar, fieldTypeString, fieldTypeVarString, fieldTypeTinyBLOB, - fieldTypeMediumBLOB, fieldTypeLongBLOB, fieldTypeBLOB: - // charsetnr == 63 means this column is binary. - // https://dev.mysql.com/doc/refman/8.0/en/c-api-data-structures.html - if mf.charSet == 63 { - return scanTypeRawBytes - } - if mf.flags&flagNotNULL != 0 { - return scanTypeString - } - return scanTypeNullString - - case fieldTypeDecimal, fieldTypeNewDecimal, fieldTypeBit, - fieldTypeEnum, fieldTypeSet, - fieldTypeGeometry, fieldTypeJSON, fieldTypeTime: + case fieldTypeDecimal, fieldTypeNewDecimal, fieldTypeVarChar, + fieldTypeBit, fieldTypeEnum, fieldTypeSet, fieldTypeTinyBLOB, + fieldTypeMediumBLOB, fieldTypeLongBLOB, fieldTypeBLOB, + fieldTypeVarString, fieldTypeString, fieldTypeGeometry, fieldTypeJSON, + fieldTypeTime: return scanTypeRawBytes case fieldTypeDate, fieldTypeNewDate, From 296987f946840022593fa9179ee8cfb58307caf5 Mon Sep 17 00:00:00 2001 From: Matthew Herrmann <47012945+mherr-google@users.noreply.github.com> Date: Fri, 1 Nov 2019 17:35:43 +1100 Subject: [PATCH 38/93] Fix connection leak caused by rapid context cancellation (#1024) Fixes #1023 --- connector.go | 1 + driver_go110_test.go | 53 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/connector.go b/connector.go index 5aaaba43e..bbad4e23b 100644 --- a/connector.go +++ b/connector.go @@ -64,6 +64,7 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { // Call startWatcher for context support (From Go 1.8) mc.startWatcher() if err := mc.watchCancel(ctx); err != nil { + mc.cleanup() return nil, err } defer mc.finish() diff --git a/driver_go110_test.go b/driver_go110_test.go index 19a0e5956..fd8df8975 100644 --- a/driver_go110_test.go +++ b/driver_go110_test.go @@ -135,3 +135,56 @@ func TestConnectorTimeoutsDuringOpen(t *testing.T) { t.Fatalf("(*Connector).Connect should have timed out") } } + +// A connection which can only be closed. +type dummyConnection struct { + net.Conn + closed bool +} + +func (d *dummyConnection) Close() error { + d.closed = true + return nil +} + +func TestConnectorTimeoutsWatchCancel(t *testing.T) { + var ( + cancel func() // Used to cancel the context just after connecting. + created *dummyConnection // The created connection. + ) + + RegisterDialContext("TestConnectorTimeoutsWatchCancel", func(ctx context.Context, addr string) (net.Conn, error) { + // Canceling at this time triggers the watchCancel error branch in Connect(). + cancel() + created = &dummyConnection{} + return created, nil + }) + + mycnf := NewConfig() + mycnf.User = "root" + mycnf.Addr = "foo" + mycnf.Net = "TestConnectorTimeoutsWatchCancel" + + conn, err := NewConnector(mycnf) + if err != nil { + t.Fatal(err) + } + + db := sql.OpenDB(conn) + defer db.Close() + + var ctx context.Context + ctx, cancel = context.WithCancel(context.Background()) + defer cancel() + + if _, err := db.Conn(ctx); err != context.Canceled { + t.Errorf("got %v, want context.Canceled", err) + } + + if created == nil { + t.Fatal("no connection created") + } + if !created.closed { + t.Errorf("connection not closed") + } +} From b57978c556ea168c02a3246eb044fd8b5afa5320 Mon Sep 17 00:00:00 2001 From: Bouke van der Bijl Date: Fri, 8 Nov 2019 14:06:54 +0100 Subject: [PATCH 39/93] connector: don't return ErrBadConn when failing to connect (#1020) ErrBadConn should only be returned for an already established connection, not when creating a new one. --- AUTHORS | 1 + connector.go | 4 ---- connector_test.go | 30 ++++++++++++++++++++++++++++++ driver_test.go | 2 +- 4 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 connector_test.go diff --git a/AUTHORS b/AUTHORS index d87414e82..9765b5348 100644 --- a/AUTHORS +++ b/AUTHORS @@ -91,6 +91,7 @@ Zhenye Xie Barracuda Networks, Inc. Counting Ltd. +DigitalOcean Inc. Facebook Inc. GitHub Inc. Google Inc. diff --git a/connector.go b/connector.go index bbad4e23b..eac0f01aa 100644 --- a/connector.go +++ b/connector.go @@ -44,10 +44,6 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { } if err != nil { - if nerr, ok := err.(net.Error); ok && nerr.Temporary() { - errLog.Print("net.Error from Dial()': ", nerr.Error()) - return nil, driver.ErrBadConn - } return nil, err } diff --git a/connector_test.go b/connector_test.go new file mode 100644 index 000000000..976903c5b --- /dev/null +++ b/connector_test.go @@ -0,0 +1,30 @@ +package mysql + +import ( + "context" + "net" + "testing" + "time" +) + +func TestConnectorReturnsTimeout(t *testing.T) { + connector := &connector{&Config{ + Net: "tcp", + Addr: "1.1.1.1:1234", + Timeout: 10 * time.Millisecond, + }} + + _, err := connector.Connect(context.Background()) + if err == nil { + t.Fatal("error expected") + } + + if nerr, ok := err.(*net.OpError); ok { + expected := "dial tcp 1.1.1.1:1234: i/o timeout" + if nerr.Error() != expected { + t.Fatalf("expected %q, got %q", expected, nerr.Error()) + } + } else { + t.Fatalf("expected %T, got %T", nerr, err) + } +} diff --git a/driver_test.go b/driver_test.go index 3dee1bab2..df7353dbb 100644 --- a/driver_test.go +++ b/driver_test.go @@ -1874,7 +1874,7 @@ func TestDialNonRetryableNetErr(t *testing.T) { func TestDialTemporaryNetErr(t *testing.T) { testErr := netErrorMock{temporary: true} - testDialError(t, testErr, driver.ErrBadConn) + testDialError(t, testErr, testErr) } // Tests custom dial functions From b4242bab7dc57d57fef955900943999b6fb34b67 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 14 Nov 2019 20:57:53 +0900 Subject: [PATCH 40/93] Return ErrBadConn in Prepare (#1030) --- connection.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/connection.go b/connection.go index 565a5480a..e4bb59e67 100644 --- a/connection.go +++ b/connection.go @@ -154,7 +154,9 @@ func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) { // Send command err := mc.writeCommandPacketStr(comStmtPrepare, query) if err != nil { - return nil, mc.markBadConn(err) + // STMT_PREPARE is safe to retry. So we can return ErrBadConn here. + errLog.Print(err) + return nil, driver.ErrBadConn } stmt := &mysqlStmt{ From 15462c1d60d42ecca11d6ef9fec0b0afd5833459 Mon Sep 17 00:00:00 2001 From: zjj Date: Thu, 21 Nov 2019 14:26:41 +0800 Subject: [PATCH 41/93] Set timeout for custom dialer. (#1035) Fixes #1034. --- AUTHORS | 1 + connector.go | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 9765b5348..7d647012d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -44,6 +44,7 @@ James Harr Jeff Hodges Jeffrey Charles Jerome Meyer +Jiajia Zhong Jian Zhen Joshua Prunier Julien Lefevre diff --git a/connector.go b/connector.go index eac0f01aa..d567b4e4f 100644 --- a/connector.go +++ b/connector.go @@ -37,7 +37,13 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { dial, ok := dials[mc.cfg.Net] dialsLock.RUnlock() if ok { - mc.netConn, err = dial(ctx, mc.cfg.Addr) + dctx := ctx + if mc.cfg.Timeout > 0 { + var cancel context.CancelFunc + dctx, cancel = context.WithTimeout(ctx, c.cfg.Timeout) + defer cancel() + } + mc.netConn, err = dial(dctx, mc.cfg.Addr) } else { nd := net.Dialer{Timeout: mc.cfg.Timeout} mc.netConn, err = nd.DialContext(ctx, mc.cfg.Net, mc.cfg.Addr) From 94084c99124c2870fa9c72c4f5e41b8d2d8335a1 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 12 Dec 2019 00:38:32 +0900 Subject: [PATCH 42/93] Drop Go 1.9 support (#1017) * Drop Go 1.9 support * Add test for Go 1.13. --- .travis.yml | 2 +- README.md | 2 +- go.mod | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9d3139207..56fcf25f2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,10 @@ sudo: false language: go go: - - 1.9.x - 1.10.x - 1.11.x - 1.12.x + - 1.13.x - master before_install: diff --git a/README.md b/README.md index dce7a5325..2d15ffda3 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) pac * Optional placeholder interpolation ## Requirements - * Go 1.9 or higher. We aim to support the 3 latest versions of Go. + * Go 1.10 or higher. We aim to support the 3 latest versions of Go. * MySQL (4.1+), MariaDB, Percona Server, Google CloudSQL or Sphinx (2.2.3+) --------------------------------------- diff --git a/go.mod b/go.mod index 29e5d800d..fffbf6a90 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/go-sql-driver/mysql -go 1.9 +go 1.10 From b66d043e6c8986ca01241b990326db395f9c0afd Mon Sep 17 00:00:00 2001 From: Nathanial Murphy Date: Thu, 12 Dec 2019 10:19:55 +1000 Subject: [PATCH 43/93] Remove "go1.10" build tag (#1016) Some IDEs and editors refuse to acknowledge the "go1.10" build tag when autocompleting & compiling. Removing said tag increases usibility of the library for those stuck with these editors. --- AUTHORS | 1 + driver.go | 22 +++++ driver_go110.go | 37 --------- driver_go110_test.go | 190 ------------------------------------------- driver_test.go | 169 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 192 insertions(+), 227 deletions(-) delete mode 100644 driver_go110.go delete mode 100644 driver_go110_test.go diff --git a/AUTHORS b/AUTHORS index 7d647012d..ad5989800 100644 --- a/AUTHORS +++ b/AUTHORS @@ -63,6 +63,7 @@ Lucas Liu Luke Scott Maciej Zimnoch Michael Woolnough +Nathanial Murphy Nicola Peduzzi Olivier Mengué oscarzhao diff --git a/driver.go b/driver.go index 1f9decf80..c1bdf1199 100644 --- a/driver.go +++ b/driver.go @@ -83,3 +83,25 @@ func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { func init() { sql.Register("mysql", &MySQLDriver{}) } + +// NewConnector returns new driver.Connector. +func NewConnector(cfg *Config) (driver.Connector, error) { + cfg = cfg.Clone() + // normalize the contents of cfg so calls to NewConnector have the same + // behavior as MySQLDriver.OpenConnector + if err := cfg.normalize(); err != nil { + return nil, err + } + return &connector{cfg: cfg}, nil +} + +// OpenConnector implements driver.DriverContext. +func (d MySQLDriver) OpenConnector(dsn string) (driver.Connector, error) { + cfg, err := ParseDSN(dsn) + if err != nil { + return nil, err + } + return &connector{ + cfg: cfg, + }, nil +} diff --git a/driver_go110.go b/driver_go110.go deleted file mode 100644 index eb5a8fe9b..000000000 --- a/driver_go110.go +++ /dev/null @@ -1,37 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -// +build go1.10 - -package mysql - -import ( - "database/sql/driver" -) - -// NewConnector returns new driver.Connector. -func NewConnector(cfg *Config) (driver.Connector, error) { - cfg = cfg.Clone() - // normalize the contents of cfg so calls to NewConnector have the same - // behavior as MySQLDriver.OpenConnector - if err := cfg.normalize(); err != nil { - return nil, err - } - return &connector{cfg: cfg}, nil -} - -// OpenConnector implements driver.DriverContext. -func (d MySQLDriver) OpenConnector(dsn string) (driver.Connector, error) { - cfg, err := ParseDSN(dsn) - if err != nil { - return nil, err - } - return &connector{ - cfg: cfg, - }, nil -} diff --git a/driver_go110_test.go b/driver_go110_test.go deleted file mode 100644 index fd8df8975..000000000 --- a/driver_go110_test.go +++ /dev/null @@ -1,190 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -// +build go1.10 - -package mysql - -import ( - "context" - "database/sql" - "database/sql/driver" - "fmt" - "net" - "testing" - "time" -) - -var _ driver.DriverContext = &MySQLDriver{} - -type dialCtxKey struct{} - -func TestConnectorObeysDialTimeouts(t *testing.T) { - if !available { - t.Skipf("MySQL server not running on %s", netAddr) - } - - RegisterDialContext("dialctxtest", func(ctx context.Context, addr string) (net.Conn, error) { - var d net.Dialer - if !ctx.Value(dialCtxKey{}).(bool) { - return nil, fmt.Errorf("test error: query context is not propagated to our dialer") - } - return d.DialContext(ctx, prot, addr) - }) - - db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@dialctxtest(%s)/%s?timeout=30s", user, pass, addr, dbname)) - if err != nil { - t.Fatalf("error connecting: %s", err.Error()) - } - defer db.Close() - - ctx := context.WithValue(context.Background(), dialCtxKey{}, true) - - _, err = db.ExecContext(ctx, "DO 1") - if err != nil { - t.Fatal(err) - } -} - -func configForTests(t *testing.T) *Config { - if !available { - t.Skipf("MySQL server not running on %s", netAddr) - } - - mycnf := NewConfig() - mycnf.User = user - mycnf.Passwd = pass - mycnf.Addr = addr - mycnf.Net = prot - mycnf.DBName = dbname - return mycnf -} - -func TestNewConnector(t *testing.T) { - mycnf := configForTests(t) - conn, err := NewConnector(mycnf) - if err != nil { - t.Fatal(err) - } - - db := sql.OpenDB(conn) - defer db.Close() - - if err := db.Ping(); err != nil { - t.Fatal(err) - } -} - -type slowConnection struct { - net.Conn - slowdown time.Duration -} - -func (sc *slowConnection) Read(b []byte) (int, error) { - time.Sleep(sc.slowdown) - return sc.Conn.Read(b) -} - -type connectorHijack struct { - driver.Connector - connErr error -} - -func (cw *connectorHijack) Connect(ctx context.Context) (driver.Conn, error) { - var conn driver.Conn - conn, cw.connErr = cw.Connector.Connect(ctx) - return conn, cw.connErr -} - -func TestConnectorTimeoutsDuringOpen(t *testing.T) { - RegisterDialContext("slowconn", func(ctx context.Context, addr string) (net.Conn, error) { - var d net.Dialer - conn, err := d.DialContext(ctx, prot, addr) - if err != nil { - return nil, err - } - return &slowConnection{Conn: conn, slowdown: 100 * time.Millisecond}, nil - }) - - mycnf := configForTests(t) - mycnf.Net = "slowconn" - - conn, err := NewConnector(mycnf) - if err != nil { - t.Fatal(err) - } - - hijack := &connectorHijack{Connector: conn} - - db := sql.OpenDB(hijack) - defer db.Close() - - ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) - defer cancel() - - _, err = db.ExecContext(ctx, "DO 1") - if err != context.DeadlineExceeded { - t.Fatalf("ExecContext should have timed out") - } - if hijack.connErr != context.DeadlineExceeded { - t.Fatalf("(*Connector).Connect should have timed out") - } -} - -// A connection which can only be closed. -type dummyConnection struct { - net.Conn - closed bool -} - -func (d *dummyConnection) Close() error { - d.closed = true - return nil -} - -func TestConnectorTimeoutsWatchCancel(t *testing.T) { - var ( - cancel func() // Used to cancel the context just after connecting. - created *dummyConnection // The created connection. - ) - - RegisterDialContext("TestConnectorTimeoutsWatchCancel", func(ctx context.Context, addr string) (net.Conn, error) { - // Canceling at this time triggers the watchCancel error branch in Connect(). - cancel() - created = &dummyConnection{} - return created, nil - }) - - mycnf := NewConfig() - mycnf.User = "root" - mycnf.Addr = "foo" - mycnf.Net = "TestConnectorTimeoutsWatchCancel" - - conn, err := NewConnector(mycnf) - if err != nil { - t.Fatal(err) - } - - db := sql.OpenDB(conn) - defer db.Close() - - var ctx context.Context - ctx, cancel = context.WithCancel(context.Background()) - defer cancel() - - if _, err := db.Conn(ctx); err != context.Canceled { - t.Errorf("got %v, want context.Canceled", err) - } - - if created == nil { - t.Fatal("no connection created") - } - if !created.closed { - t.Errorf("connection not closed") - } -} diff --git a/driver_test.go b/driver_test.go index df7353dbb..ace083dfc 100644 --- a/driver_test.go +++ b/driver_test.go @@ -2994,3 +2994,172 @@ func TestRawBytesAreNotModified(t *testing.T) { } }) } + +var _ driver.DriverContext = &MySQLDriver{} + +type dialCtxKey struct{} + +func TestConnectorObeysDialTimeouts(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + RegisterDialContext("dialctxtest", func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + if !ctx.Value(dialCtxKey{}).(bool) { + return nil, fmt.Errorf("test error: query context is not propagated to our dialer") + } + return d.DialContext(ctx, prot, addr) + }) + + db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@dialctxtest(%s)/%s?timeout=30s", user, pass, addr, dbname)) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + ctx := context.WithValue(context.Background(), dialCtxKey{}, true) + + _, err = db.ExecContext(ctx, "DO 1") + if err != nil { + t.Fatal(err) + } +} + +func configForTests(t *testing.T) *Config { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + mycnf := NewConfig() + mycnf.User = user + mycnf.Passwd = pass + mycnf.Addr = addr + mycnf.Net = prot + mycnf.DBName = dbname + return mycnf +} + +func TestNewConnector(t *testing.T) { + mycnf := configForTests(t) + conn, err := NewConnector(mycnf) + if err != nil { + t.Fatal(err) + } + + db := sql.OpenDB(conn) + defer db.Close() + + if err := db.Ping(); err != nil { + t.Fatal(err) + } +} + +type slowConnection struct { + net.Conn + slowdown time.Duration +} + +func (sc *slowConnection) Read(b []byte) (int, error) { + time.Sleep(sc.slowdown) + return sc.Conn.Read(b) +} + +type connectorHijack struct { + driver.Connector + connErr error +} + +func (cw *connectorHijack) Connect(ctx context.Context) (driver.Conn, error) { + var conn driver.Conn + conn, cw.connErr = cw.Connector.Connect(ctx) + return conn, cw.connErr +} + +func TestConnectorTimeoutsDuringOpen(t *testing.T) { + RegisterDialContext("slowconn", func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + conn, err := d.DialContext(ctx, prot, addr) + if err != nil { + return nil, err + } + return &slowConnection{Conn: conn, slowdown: 100 * time.Millisecond}, nil + }) + + mycnf := configForTests(t) + mycnf.Net = "slowconn" + + conn, err := NewConnector(mycnf) + if err != nil { + t.Fatal(err) + } + + hijack := &connectorHijack{Connector: conn} + + db := sql.OpenDB(hijack) + defer db.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + _, err = db.ExecContext(ctx, "DO 1") + if err != context.DeadlineExceeded { + t.Fatalf("ExecContext should have timed out") + } + if hijack.connErr != context.DeadlineExceeded { + t.Fatalf("(*Connector).Connect should have timed out") + } +} + +// A connection which can only be closed. +type dummyConnection struct { + net.Conn + closed bool +} + +func (d *dummyConnection) Close() error { + d.closed = true + return nil +} + +func TestConnectorTimeoutsWatchCancel(t *testing.T) { + var ( + cancel func() // Used to cancel the context just after connecting. + created *dummyConnection // The created connection. + ) + + RegisterDialContext("TestConnectorTimeoutsWatchCancel", func(ctx context.Context, addr string) (net.Conn, error) { + // Canceling at this time triggers the watchCancel error branch in Connect(). + cancel() + created = &dummyConnection{} + return created, nil + }) + + mycnf := NewConfig() + mycnf.User = "root" + mycnf.Addr = "foo" + mycnf.Net = "TestConnectorTimeoutsWatchCancel" + + conn, err := NewConnector(mycnf) + if err != nil { + t.Fatal(err) + } + + db := sql.OpenDB(conn) + defer db.Close() + + var ctx context.Context + ctx, cancel = context.WithCancel(context.Background()) + defer cancel() + + if _, err := db.Conn(ctx); err != context.Canceled { + t.Errorf("got %v, want context.Canceled", err) + } + + if created == nil { + t.Fatal("no connection created") + } + if !created.closed { + t.Errorf("connection not closed") + } +} From 87239404061139bfbd9a61b167c5ed54539e6086 Mon Sep 17 00:00:00 2001 From: Ichinose Shogo Date: Sun, 5 Jan 2020 05:18:52 +0000 Subject: [PATCH 44/93] fix compile error of connCheck.go (#1048) * fix compile error of connCheck.go * remove "appengine" build tag --- conncheck.go | 2 +- conncheck_dummy.go | 2 +- conncheck_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conncheck.go b/conncheck.go index 70e9925f6..3dd115485 100644 --- a/conncheck.go +++ b/conncheck.go @@ -6,7 +6,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. -// +build !windows,!appengine +// +build linux darwin package mysql diff --git a/conncheck_dummy.go b/conncheck_dummy.go index 4888288aa..d042811c1 100644 --- a/conncheck_dummy.go +++ b/conncheck_dummy.go @@ -6,7 +6,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. -// +build windows appengine +// +build !linux,!darwin package mysql diff --git a/conncheck_test.go b/conncheck_test.go index b7234b0f5..8cd81223a 100644 --- a/conncheck_test.go +++ b/conncheck_test.go @@ -6,7 +6,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. -// +build go1.10,!windows +// +build linux darwin package mysql From 4bdaef4517f923802be30b39b8c12d97fee32466 Mon Sep 17 00:00:00 2001 From: Julien Schmidt Date: Tue, 7 Jan 2020 08:34:00 +0100 Subject: [PATCH 45/93] conncheck: build on more supported platforms (#1051) --- conncheck.go | 2 +- conncheck_dummy.go | 2 +- conncheck_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conncheck.go b/conncheck.go index 3dd115485..024eb2858 100644 --- a/conncheck.go +++ b/conncheck.go @@ -6,7 +6,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. -// +build linux darwin +// +build linux darwin dragonfly freebsd netbsd openbsd solaris illumos package mysql diff --git a/conncheck_dummy.go b/conncheck_dummy.go index d042811c1..ea7fb607a 100644 --- a/conncheck_dummy.go +++ b/conncheck_dummy.go @@ -6,7 +6,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. -// +build !linux,!darwin +// +build !linux,!darwin,!dragonfly,!freebsd,!netbsd,!openbsd,!solaris,!illumos package mysql diff --git a/conncheck_test.go b/conncheck_test.go index 8cd81223a..53995517b 100644 --- a/conncheck_test.go +++ b/conncheck_test.go @@ -6,7 +6,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. -// +build linux darwin +// +build linux darwin dragonfly freebsd netbsd openbsd solaris illumos package mysql From 6844171c7b652e0a8271a909fc48f6a32492d04f Mon Sep 17 00:00:00 2001 From: Julien Schmidt Date: Tue, 7 Jan 2020 12:54:56 +0100 Subject: [PATCH 46/93] conncheck: allow to disable via config (#1052) * conncheck: allow to disable via config * dsn: refactor writing of params in FormatDSN --- README.md | 11 ++++ dsn.go | 166 ++++++++++++++-------------------------------------- dsn_test.go | 36 ++++++------ packets.go | 2 +- 4 files changed, 75 insertions(+), 140 deletions(-) diff --git a/README.md b/README.md index 2d15ffda3..239de7b6b 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,17 @@ Sets the charset used for client-server interaction (`"SET NAMES "`). If Usage of the `charset` parameter is discouraged because it issues additional queries to the server. Unless you need the fallback behavior, please use `collation` instead. +##### `checkConnLiveness` + +``` +Type: bool +Valid Values: true, false +Default: true +``` + +On supported platforms connections retrieved from the connection pool are checked for liveness before using them. If the check fails, the respective connection is marked as bad and the query retried with another connection. +`checkConnLiveness=false` disables this liveness check of connections. + ##### `collation` ``` diff --git a/dsn.go b/dsn.go index 1d9b4ab0a..75c8c2489 100644 --- a/dsn.go +++ b/dsn.go @@ -55,6 +55,7 @@ type Config struct { AllowCleartextPasswords bool // Allows the cleartext client side plugin AllowNativePasswords bool // Allows the native password authentication method AllowOldPasswords bool // Allows the old insecure password method + CheckConnLiveness bool // Check connections for liveness before using them ClientFoundRows bool // Return number of matching rows instead of rows changed ColumnsWithAlias bool // Prepend table alias to column names InterpolateParams bool // Interpolate placeholders into query string @@ -70,6 +71,7 @@ func NewConfig() *Config { Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, + CheckConnLiveness: true, } } @@ -148,6 +150,19 @@ func (cfg *Config) normalize() error { return nil } +func writeDSNParam(buf *bytes.Buffer, hasParam *bool, name, value string) { + buf.Grow(1 + len(name) + 1 + len(value)) + if !*hasParam { + *hasParam = true + buf.WriteByte('?') + } else { + buf.WriteByte('&') + } + buf.WriteString(name) + buf.WriteByte('=') + buf.WriteString(value) +} + // FormatDSN formats the given Config into a DSN string which can be passed to // the driver. func (cfg *Config) FormatDSN() string { @@ -186,165 +201,75 @@ func (cfg *Config) FormatDSN() string { } if cfg.AllowCleartextPasswords { - if hasParam { - buf.WriteString("&allowCleartextPasswords=true") - } else { - hasParam = true - buf.WriteString("?allowCleartextPasswords=true") - } + writeDSNParam(&buf, &hasParam, "allowCleartextPasswords", "true") } if !cfg.AllowNativePasswords { - if hasParam { - buf.WriteString("&allowNativePasswords=false") - } else { - hasParam = true - buf.WriteString("?allowNativePasswords=false") - } + writeDSNParam(&buf, &hasParam, "allowNativePasswords", "false") } if cfg.AllowOldPasswords { - if hasParam { - buf.WriteString("&allowOldPasswords=true") - } else { - hasParam = true - buf.WriteString("?allowOldPasswords=true") - } + writeDSNParam(&buf, &hasParam, "allowOldPasswords", "true") + } + + if !cfg.CheckConnLiveness { + writeDSNParam(&buf, &hasParam, "checkConnLiveness", "false") } if cfg.ClientFoundRows { - if hasParam { - buf.WriteString("&clientFoundRows=true") - } else { - hasParam = true - buf.WriteString("?clientFoundRows=true") - } + writeDSNParam(&buf, &hasParam, "clientFoundRows", "true") } if col := cfg.Collation; col != defaultCollation && len(col) > 0 { - if hasParam { - buf.WriteString("&collation=") - } else { - hasParam = true - buf.WriteString("?collation=") - } - buf.WriteString(col) + writeDSNParam(&buf, &hasParam, "collation", col) } if cfg.ColumnsWithAlias { - if hasParam { - buf.WriteString("&columnsWithAlias=true") - } else { - hasParam = true - buf.WriteString("?columnsWithAlias=true") - } + writeDSNParam(&buf, &hasParam, "columnsWithAlias", "true") } if cfg.InterpolateParams { - if hasParam { - buf.WriteString("&interpolateParams=true") - } else { - hasParam = true - buf.WriteString("?interpolateParams=true") - } + writeDSNParam(&buf, &hasParam, "interpolateParams", "true") } if cfg.Loc != time.UTC && cfg.Loc != nil { - if hasParam { - buf.WriteString("&loc=") - } else { - hasParam = true - buf.WriteString("?loc=") - } - buf.WriteString(url.QueryEscape(cfg.Loc.String())) + writeDSNParam(&buf, &hasParam, "loc", url.QueryEscape(cfg.Loc.String())) } if cfg.MultiStatements { - if hasParam { - buf.WriteString("&multiStatements=true") - } else { - hasParam = true - buf.WriteString("?multiStatements=true") - } + writeDSNParam(&buf, &hasParam, "multiStatements", "true") } if cfg.ParseTime { - if hasParam { - buf.WriteString("&parseTime=true") - } else { - hasParam = true - buf.WriteString("?parseTime=true") - } + writeDSNParam(&buf, &hasParam, "parseTime", "true") } if cfg.ReadTimeout > 0 { - if hasParam { - buf.WriteString("&readTimeout=") - } else { - hasParam = true - buf.WriteString("?readTimeout=") - } - buf.WriteString(cfg.ReadTimeout.String()) + writeDSNParam(&buf, &hasParam, "readTimeout", cfg.ReadTimeout.String()) } if cfg.RejectReadOnly { - if hasParam { - buf.WriteString("&rejectReadOnly=true") - } else { - hasParam = true - buf.WriteString("?rejectReadOnly=true") - } + writeDSNParam(&buf, &hasParam, "rejectReadOnly", "true") } if len(cfg.ServerPubKey) > 0 { - if hasParam { - buf.WriteString("&serverPubKey=") - } else { - hasParam = true - buf.WriteString("?serverPubKey=") - } - buf.WriteString(url.QueryEscape(cfg.ServerPubKey)) + writeDSNParam(&buf, &hasParam, "serverPubKey", url.QueryEscape(cfg.ServerPubKey)) } if cfg.Timeout > 0 { - if hasParam { - buf.WriteString("&timeout=") - } else { - hasParam = true - buf.WriteString("?timeout=") - } - buf.WriteString(cfg.Timeout.String()) + writeDSNParam(&buf, &hasParam, "timeout", cfg.Timeout.String()) } if len(cfg.TLSConfig) > 0 { - if hasParam { - buf.WriteString("&tls=") - } else { - hasParam = true - buf.WriteString("?tls=") - } - buf.WriteString(url.QueryEscape(cfg.TLSConfig)) + writeDSNParam(&buf, &hasParam, "tls", url.QueryEscape(cfg.TLSConfig)) } if cfg.WriteTimeout > 0 { - if hasParam { - buf.WriteString("&writeTimeout=") - } else { - hasParam = true - buf.WriteString("?writeTimeout=") - } - buf.WriteString(cfg.WriteTimeout.String()) + writeDSNParam(&buf, &hasParam, "writeTimeout", cfg.WriteTimeout.String()) } if cfg.MaxAllowedPacket != defaultMaxAllowedPacket { - if hasParam { - buf.WriteString("&maxAllowedPacket=") - } else { - hasParam = true - buf.WriteString("?maxAllowedPacket=") - } - buf.WriteString(strconv.Itoa(cfg.MaxAllowedPacket)) - + writeDSNParam(&buf, &hasParam, "maxAllowedPacket", strconv.Itoa(cfg.MaxAllowedPacket)) } // other params @@ -355,16 +280,7 @@ func (cfg *Config) FormatDSN() string { } sort.Strings(params) for _, param := range params { - if hasParam { - buf.WriteByte('&') - } else { - hasParam = true - buf.WriteByte('?') - } - - buf.WriteString(param) - buf.WriteByte('=') - buf.WriteString(url.QueryEscape(cfg.Params[param])) + writeDSNParam(&buf, &hasParam, param, url.QueryEscape(cfg.Params[param])) } } @@ -491,6 +407,14 @@ func parseDSNParams(cfg *Config, params string) (err error) { return errors.New("invalid bool value: " + value) } + // Check connections for Liveness before using them + case "checkConnLiveness": + var isBool bool + cfg.CheckConnLiveness, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + // Switch "rowsAffected" mode case "clientFoundRows": var isBool bool diff --git a/dsn_test.go b/dsn_test.go index 50dc2932c..89815b341 100644 --- a/dsn_test.go +++ b/dsn_test.go @@ -22,55 +22,55 @@ var testDSNs = []struct { out *Config }{{ "username:password@protocol(address)/dbname?param=value", - &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "username:password@protocol(address)/dbname?param=value&columnsWithAlias=true", - &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, ColumnsWithAlias: true}, + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, ColumnsWithAlias: true}, }, { "username:password@protocol(address)/dbname?param=value&columnsWithAlias=true&multiStatements=true", - &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, ColumnsWithAlias: true, MultiStatements: true}, + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, ColumnsWithAlias: true, MultiStatements: true}, }, { "user@unix(/path/to/socket)/dbname?charset=utf8", - &Config{User: "user", Net: "unix", Addr: "/path/to/socket", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{User: "user", Net: "unix", Addr: "/path/to/socket", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "user:password@tcp(localhost:5555)/dbname?charset=utf8&tls=true", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, TLSConfig: "true"}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, TLSConfig: "true"}, }, { "user:password@tcp(localhost:5555)/dbname?charset=utf8mb4,utf8&tls=skip-verify", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8mb4,utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, TLSConfig: "skip-verify"}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8mb4,utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, TLSConfig: "skip-verify"}, }, { "user:password@/dbname?loc=UTC&timeout=30s&readTimeout=1s&writeTimeout=1s&allowAllFiles=1&clientFoundRows=true&allowOldPasswords=TRUE&collation=utf8mb4_unicode_ci&maxAllowedPacket=16777216&tls=false&allowCleartextPasswords=true&parseTime=true&rejectReadOnly=true", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_unicode_ci", Loc: time.UTC, TLSConfig: "false", AllowCleartextPasswords: true, AllowNativePasswords: true, Timeout: 30 * time.Second, ReadTimeout: time.Second, WriteTimeout: time.Second, AllowAllFiles: true, AllowOldPasswords: true, ClientFoundRows: true, MaxAllowedPacket: 16777216, ParseTime: true, RejectReadOnly: true}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_unicode_ci", Loc: time.UTC, TLSConfig: "false", AllowCleartextPasswords: true, AllowNativePasswords: true, Timeout: 30 * time.Second, ReadTimeout: time.Second, WriteTimeout: time.Second, AllowAllFiles: true, AllowOldPasswords: true, CheckConnLiveness: true, ClientFoundRows: true, MaxAllowedPacket: 16777216, ParseTime: true, RejectReadOnly: true}, }, { - "user:password@/dbname?allowNativePasswords=false&maxAllowedPacket=0", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, AllowNativePasswords: false}, + "user:password@/dbname?allowNativePasswords=false&checkConnLiveness=false&maxAllowedPacket=0", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, AllowNativePasswords: false, CheckConnLiveness: false}, }, { "user:p@ss(word)@tcp([de:ad:be:ef::ca:fe]:80)/dbname?loc=Local", - &Config{User: "user", Passwd: "p@ss(word)", Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:80", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.Local, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{User: "user", Passwd: "p@ss(word)", Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:80", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.Local, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "/dbname", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "@/", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "/", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "user:p@/ssword@/", - &Config{User: "user", Passwd: "p@/ssword", Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{User: "user", Passwd: "p@/ssword", Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "unix/?arg=%2Fsome%2Fpath.ext", - &Config{Net: "unix", Addr: "/tmp/mysql.sock", Params: map[string]string{"arg": "/some/path.ext"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "unix", Addr: "/tmp/mysql.sock", Params: map[string]string{"arg": "/some/path.ext"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "tcp(127.0.0.1)/dbname", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "tcp(de:ad:be:ef::ca:fe)/dbname", - &Config{Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, } diff --git a/packets.go b/packets.go index 30b3352c2..82ad7a200 100644 --- a/packets.go +++ b/packets.go @@ -115,7 +115,7 @@ func (mc *mysqlConn) writePacket(data []byte) error { if mc.cfg.ReadTimeout != 0 { err = conn.SetReadDeadline(time.Time{}) } - if err == nil { + if err == nil && mc.cfg.CheckConnLiveness { err = connCheck(conn) } if err != nil { From 2898b563649417674caafd1a7ee2acf21d06a3ce Mon Sep 17 00:00:00 2001 From: Julien Schmidt Date: Tue, 7 Jan 2020 17:19:51 +0100 Subject: [PATCH 47/93] README: update default collation to utf8mb4_general_ci (#1054) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 239de7b6b..d2627a41a 100644 --- a/README.md +++ b/README.md @@ -463,13 +463,13 @@ Alternatively you can use the [`NullTime`](https://godoc.org/github.com/go-sql-d ### Unicode support -Since version 1.1 Go-MySQL-Driver automatically uses the collation `utf8_general_ci` by default. +Since version 1.5 Go-MySQL-Driver automatically uses the collation ` utf8mb4_general_ci` by default. Other collations / charsets can be set using the [`collation`](#collation) DSN parameter. Version 1.0 of the driver recommended adding `&charset=utf8` (alias for `SET NAMES utf8`) to the DSN to enable proper UTF-8 support. This is not necessary anymore. The [`collation`](#collation) parameter should be preferred to set another collation / charset than the default. -See http://dev.mysql.com/doc/refman/5.7/en/charset-unicode.html for more details on MySQL's Unicode support. +See http://dev.mysql.com/doc/refman/8.0/en/charset-unicode.html for more details on MySQL's Unicode support. ## Testing / Development To run the driver tests you may need to adjust the configuration. See the [Testing Wiki-Page](https://github.com/go-sql-driver/mysql/wiki/Testing "Testing") for details. From 17ef3dd9d98b69acec3e85878995ada9533a9370 Mon Sep 17 00:00:00 2001 From: Julien Schmidt Date: Tue, 7 Jan 2020 17:33:55 +0100 Subject: [PATCH 48/93] Release v1.5.0 (#1047) * CHANGELOG: include v1.4.1 * Release v1.5.0 --- CHANGELOG.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d87d74c9..9cb97b38d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,42 @@ +## Version 1.5 (2020-01-07) + +Changes: + + - Dropped support Go 1.9 and lower (#823, #829, #886, #1016, #1017) + - Improve buffer handling (#890) + - Document potentially insecure TLS configs (#901) + - Use a double-buffering scheme to prevent data races (#943) + - Pass uint64 values without converting them to string (#838, #955) + - Update collations and make utf8mb4 default (#877, #1054) + - Make NullTime compatible with sql.NullTime in Go 1.13+ (#995) + - Removed CloudSQL support (#993, #1007) + - Add Go Module support (#1003) + +New Features: + + - Implement support of optional TLS (#900) + - Check connection liveness (#934, #964, #997, #1048, #1051, #1052) + - Implement Connector Interface (#941, #958, #1020, #1035) + +Bugfixes: + + - Mark connections as bad on error during ping (#875) + - Mark connections as bad on error during dial (#867) + - Fix connection leak caused by rapid context cancellation (#1024) + - Mark connections as bad on error during Conn.Prepare (#1030) + + +## Version 1.4.1 (2018-11-14) + +Bugfixes: + + - Fix TIME format for binary columns (#818) + - Fix handling of empty auth plugin names (#835) + - Fix caching_sha2_password with empty password (#826) + - Fix canceled context broke mysqlConn (#862) + - Fix OldAuthSwitchRequest support (#870) + - Fix Auth Response packet for cleartext password (#887) + ## Version 1.4 (2018-06-03) Changes: From c4f1976e433a0e5f4a0d612ee45c85f689076ae7 Mon Sep 17 00:00:00 2001 From: Alex Snast Date: Sun, 9 Feb 2020 16:41:49 +0200 Subject: [PATCH 49/93] connection: interpolate json.RawMessage as string (#1058) json encoded data is represented as bytes however it should be interpolated as a string Fixes #819 --- AUTHORS | 1 + connection.go | 9 +++++++++ connection_test.go | 28 ++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/AUTHORS b/AUTHORS index ad5989800..0896ba1bc 100644 --- a/AUTHORS +++ b/AUTHORS @@ -13,6 +13,7 @@ Aaron Hopkins Achille Roussel +Alex Snast Alexey Palazhchenko Andrew Reid Arne Hormann diff --git a/connection.go b/connection.go index e4bb59e67..b07cd7651 100644 --- a/connection.go +++ b/connection.go @@ -12,6 +12,7 @@ import ( "context" "database/sql" "database/sql/driver" + "encoding/json" "io" "net" "strconv" @@ -271,6 +272,14 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin } buf = append(buf, '\'') } + case json.RawMessage: + buf = append(buf, '\'') + if mc.status&statusNoBackslashEscapes == 0 { + buf = escapeBytesBackslash(buf, v) + } else { + buf = escapeBytesQuotes(buf, v) + } + buf = append(buf, '\'') case []byte: if v == nil { buf = append(buf, "NULL"...) diff --git a/connection_test.go b/connection_test.go index 19c17ff8b..a6d677308 100644 --- a/connection_test.go +++ b/connection_test.go @@ -11,6 +11,7 @@ package mysql import ( "context" "database/sql/driver" + "encoding/json" "errors" "net" "testing" @@ -36,6 +37,33 @@ func TestInterpolateParams(t *testing.T) { } } +func TestInterpolateParamsJSONRawMessage(t *testing.T) { + mc := &mysqlConn{ + buf: newBuffer(nil), + maxAllowedPacket: maxPacketSize, + cfg: &Config{ + InterpolateParams: true, + }, + } + + buf, err := json.Marshal(struct { + Value int `json:"value"` + }{Value: 42}) + if err != nil { + t.Errorf("Expected err=nil, got %#v", err) + return + } + q, err := mc.interpolateParams("SELECT ?", []driver.Value{json.RawMessage(buf)}) + if err != nil { + t.Errorf("Expected err=nil, got %#v", err) + return + } + expected := `SELECT '{\"value\":42}'` + if q != expected { + t.Errorf("Expected: %q\nGot: %q", expected, q) + } +} + func TestInterpolateParamsTooManyPlaceholders(t *testing.T) { mc := &mysqlConn{ buf: newBuffer(nil), From 5a8a207333b3cbdd6f50a31da2d448658343637e Mon Sep 17 00:00:00 2001 From: Noboru Saito Date: Fri, 14 Feb 2020 07:49:22 +0900 Subject: [PATCH 50/93] Fix link(#1061) (#1062) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d2627a41a..6ac81d1e4 100644 --- a/README.md +++ b/README.md @@ -477,7 +477,7 @@ To run the driver tests you may need to adjust the configuration. See the [Testi Go-MySQL-Driver is not feature-complete yet. Your help is very appreciated. If you want to contribute, you can work on an [open issue](https://github.com/go-sql-driver/mysql/issues?state=open) or review a [pull request](https://github.com/go-sql-driver/mysql/pulls). -See the [Contribution Guidelines](https://github.com/go-sql-driver/mysql/blob/master/CONTRIBUTING.md) for details. +See the [Contribution Guidelines](https://github.com/go-sql-driver/mysql/blob/master/.github/CONTRIBUTING.md) for details. --------------------------------------- From 3d8a0293423afe714a98d549f0a8015b2d0930b7 Mon Sep 17 00:00:00 2001 From: Ariel Mashraki <7413593+a8m@users.noreply.github.com> Date: Tue, 18 Feb 2020 17:16:20 +0200 Subject: [PATCH 51/93] stmt: add json.RawMessage for converter and prepared statement (#1059) Following #1058, in order for the driver.Value to get as a json.RawMessage, the converter should accept it as a valid value, and handle it as bytes in case where interpolation is disabled --- AUTHORS | 1 + driver_test.go | 24 ++++++++++++++++++++++++ packets.go | 4 ++++ statement.go | 13 +++++++++---- statement_test.go | 15 +++++++++++++++ 5 files changed, 53 insertions(+), 4 deletions(-) diff --git a/AUTHORS b/AUTHORS index 0896ba1bc..7dafbe2ec 100644 --- a/AUTHORS +++ b/AUTHORS @@ -17,6 +17,7 @@ Alex Snast Alexey Palazhchenko Andrew Reid Arne Hormann +Ariel Mashraki Asta Xie Bulat Gaifullin Carlos Nieto diff --git a/driver_test.go b/driver_test.go index ace083dfc..8edd17c47 100644 --- a/driver_test.go +++ b/driver_test.go @@ -14,6 +14,7 @@ import ( "crypto/tls" "database/sql" "database/sql/driver" + "encoding/json" "fmt" "io" "io/ioutil" @@ -559,6 +560,29 @@ func TestRawBytes(t *testing.T) { }) } +func TestRawMessage(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + v1 := json.RawMessage("{}") + v2 := json.RawMessage("[]") + rows := dbt.mustQuery("SELECT ?, ?", v1, v2) + defer rows.Close() + if rows.Next() { + var o1, o2 json.RawMessage + if err := rows.Scan(&o1, &o2); err != nil { + dbt.Errorf("Got error: %v", err) + } + if !bytes.Equal(v1, o1) { + dbt.Errorf("expected %v, got %v", v1, o1) + } + if !bytes.Equal(v2, o2) { + dbt.Errorf("expected %v, got %v", v2, o2) + } + } else { + dbt.Errorf("no data") + } + }) +} + type testValuer struct { value string } diff --git a/packets.go b/packets.go index 82ad7a200..575202ea3 100644 --- a/packets.go +++ b/packets.go @@ -13,6 +13,7 @@ import ( "crypto/tls" "database/sql/driver" "encoding/binary" + "encoding/json" "errors" "fmt" "io" @@ -1003,6 +1004,9 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { continue } + if v, ok := arg.(json.RawMessage); ok { + arg = []byte(v) + } // cache types and values switch v := arg.(type) { case int64: diff --git a/statement.go b/statement.go index f7e370939..7c6dc1367 100644 --- a/statement.go +++ b/statement.go @@ -10,6 +10,7 @@ package mysql import ( "database/sql/driver" + "encoding/json" "fmt" "io" "reflect" @@ -129,6 +130,8 @@ func (stmt *mysqlStmt) query(args []driver.Value) (*binaryRows, error) { return rows, err } +var jsonType = reflect.TypeOf(json.RawMessage{}) + type converter struct{} // ConvertValue mirrors the reference/default converter in database/sql/driver @@ -151,7 +154,6 @@ func (c converter) ConvertValue(v interface{}) (driver.Value, error) { } return sv, nil } - rv := reflect.ValueOf(v) switch rv.Kind() { case reflect.Ptr: @@ -170,11 +172,14 @@ func (c converter) ConvertValue(v interface{}) (driver.Value, error) { case reflect.Bool: return rv.Bool(), nil case reflect.Slice: - ek := rv.Type().Elem().Kind() - if ek == reflect.Uint8 { + switch t := rv.Type(); { + case t == jsonType: + return v, nil + case t.Elem().Kind() == reflect.Uint8: return rv.Bytes(), nil + default: + return nil, fmt.Errorf("unsupported type %T, a slice of %s", v, t.Elem().Kind()) } - return nil, fmt.Errorf("unsupported type %T, a slice of %s", v, ek) case reflect.String: return rv.String(), nil } diff --git a/statement_test.go b/statement_test.go index 4b9914f8e..2cc022bf5 100644 --- a/statement_test.go +++ b/statement_test.go @@ -10,6 +10,7 @@ package mysql import ( "bytes" + "encoding/json" "testing" ) @@ -124,3 +125,17 @@ func TestConvertUnsignedIntegers(t *testing.T) { t.Fatalf("uint64 high-bit converted, got %#v %T", output, output) } } + +func TestConvertJSON(t *testing.T) { + raw := json.RawMessage("{}") + + out, err := converter{}.ConvertValue(raw) + + if err != nil { + t.Fatal("json.RawMessage was failed in convert", err) + } + + if _, ok := out.(json.RawMessage); !ok { + t.Fatalf("json.RawMessage converted, got %#v %T", out, out) + } +} From dd9d356b496cd5c37543d3dd0ffbef75714b88ec Mon Sep 17 00:00:00 2001 From: Pavel Ivanov Date: Tue, 25 Feb 2020 07:24:38 -0800 Subject: [PATCH 52/93] Put zero filler into the SSL handshake packet. (#1066) According to the linked documentation at http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::SSLRequest SSLRequest packet should have zero filler similar to the regular handshake request, but now the driver puts zeros only in the regular request. Luckily vanilla MySQL doesn't rely on this zero filler and doesn't verify its presence, thus the driver worked fine so far. But MySQL can change to rely on zeros at any point. The problem was discovered while testing against a customized MySQL. --- packets.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packets.go b/packets.go index 575202ea3..5cbd53298 100644 --- a/packets.go +++ b/packets.go @@ -349,6 +349,12 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string return errors.New("unknown collation") } + // Filler [23 bytes] (all 0x00) + pos := 13 + for ; pos < 13+23; pos++ { + data[pos] = 0 + } + // SSL Connection Request Packet // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::SSLRequest if mc.cfg.tls != nil { @@ -367,12 +373,6 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string mc.buf.nc = tlsConn } - // Filler [23 bytes] (all 0x00) - pos := 13 - for ; pos < 13+23; pos++ { - data[pos] = 0 - } - // User [null terminated string] if len(mc.cfg.User) > 0 { pos += copy(data[pos:], mc.cfg.User) From 681ffa848bae8d0d7b1a452963c0705b132308f7 Mon Sep 17 00:00:00 2001 From: Animesh Ray <5434024+rayanimesh@users.noreply.github.com> Date: Wed, 11 Mar 2020 17:02:36 +0530 Subject: [PATCH 53/93] travis: Add compile check for all supported platforms (#1070) Implements a Travis CI task that checks if the driver compiles on all platforms supported by Go. Fixes #1050 --- .travis.yml | 1 + .travis/complie_check.sh | 17 +++++++++++++++++ AUTHORS | 1 + 3 files changed, 19 insertions(+) create mode 100755 .travis/complie_check.sh diff --git a/.travis.yml b/.travis.yml index 56fcf25f2..40c230105 100644 --- a/.travis.yml +++ b/.travis.yml @@ -125,5 +125,6 @@ script: - go test -v -covermode=count -coverprofile=coverage.out - go vet ./... - .travis/gofmt.sh + - .travis/complie_check.sh after_script: - $HOME/gopath/bin/goveralls -coverprofile=coverage.out -service=travis-ci diff --git a/.travis/complie_check.sh b/.travis/complie_check.sh new file mode 100755 index 000000000..3bb3ed49d --- /dev/null +++ b/.travis/complie_check.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -e +dist_list=$(go tool dist list) + +for dist in ${dist_list}; do + GOOS=$(echo ${dist} | cut -d "/" -f 1) + GOARCH=$(echo ${dist} | cut -d "/" -f 2) + set +e + GOOS=${GOOS} GOARCH=${GOARCH} go tool compile -V > /dev/null 2>&1 + if [[ $? -ne 0 ]]; then + echo "Compile support for ${GOOS}/${GOARCH} is not provided; skipping" + continue + fi + set -e + echo "Building ${GOOS}/${GOARCH}" + GOOS=${GOOS} GOARCH=${GOARCH} go build -o /dev/null + done diff --git a/AUTHORS b/AUTHORS index 7dafbe2ec..5473a8df9 100644 --- a/AUTHORS +++ b/AUTHORS @@ -16,6 +16,7 @@ Achille Roussel Alex Snast Alexey Palazhchenko Andrew Reid +Animesh Ray Arne Hormann Ariel Mashraki Asta Xie From f070e56d1471e44025607168e66be68f9aa54ed6 Mon Sep 17 00:00:00 2001 From: Julien Schmidt Date: Fri, 8 May 2020 10:52:21 +0200 Subject: [PATCH 54/93] Travis allow Go master to fail (#1092) * travis: allow master branch to fail * travis: matrix is an alias for jobs * travis: remove obsolete sudo key * travis: remove obsolete sudo keys in jobs matrix --- .travis.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 40c230105..506b4f4fb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,3 @@ -sudo: false language: go go: - 1.10.x @@ -17,10 +16,12 @@ before_script: - .travis/wait_mysql.sh - mysql -e 'create database gotest;' -matrix: +jobs: + allow_failures: + - go: master + include: - env: DB=MYSQL8 - sudo: required dist: trusty go: 1.10.x services: @@ -40,7 +41,6 @@ matrix: - export MYSQL_TEST_CONCURRENT=1 - env: DB=MYSQL57 - sudo: required dist: trusty go: 1.10.x services: @@ -60,7 +60,6 @@ matrix: - export MYSQL_TEST_CONCURRENT=1 - env: DB=MARIA55 - sudo: required dist: trusty go: 1.10.x services: @@ -80,7 +79,6 @@ matrix: - export MYSQL_TEST_CONCURRENT=1 - env: DB=MARIA10_1 - sudo: required dist: trusty go: 1.10.x services: From 343c8030d2cbdf55d368a16ec3030981f7e755f2 Mon Sep 17 00:00:00 2001 From: Zhixin Wen Date: Sat, 9 May 2020 06:02:55 -0700 Subject: [PATCH 55/93] mysqlStmt Implements CheckNamedValue (#1090) * Add CheckNamedValue for mysqlStmt * Update AUTHORS Co-authored-by: Zhixin Wen --- AUTHORS | 1 + statement.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/AUTHORS b/AUTHORS index 5473a8df9..221f4a395 100644 --- a/AUTHORS +++ b/AUTHORS @@ -91,6 +91,7 @@ Xiangyu Hu Xiaobing Jiang Xiuming Chen Zhenye Xie +Zhixin Wen # Organizations diff --git a/statement.go b/statement.go index 7c6dc1367..d3e68112f 100644 --- a/statement.go +++ b/statement.go @@ -44,6 +44,11 @@ func (stmt *mysqlStmt) ColumnConverter(idx int) driver.ValueConverter { return converter{} } +func (stmt *mysqlStmt) CheckNamedValue(nv *driver.NamedValue) (err error) { + nv.Value, err = converter{}.ConvertValue(nv.Value) + return +} + func (stmt *mysqlStmt) Exec(args []driver.Value) (driver.Result, error) { if stmt.mc.closed.IsSet() { errLog.Print(ErrInvalidConn) From 6313f20fe43bffaa8d750d5e840ff30706a5cddc Mon Sep 17 00:00:00 2001 From: Konstantinos Tsanaktsidis Date: Tue, 12 May 2020 14:12:06 +1000 Subject: [PATCH 56/93] Fix checking cancelled connections back into the connection pool (#1095) If * BeginTx is called with a non-default isolation level, * The context is canceled before SET TRANSACTION ISOLATION LEVEL completes, then the connection: * has the cancelled property set to "context cancelled", * has the closed property set to true, and, * BeginTx returns "context canceled" Because of this, the connection gets put back into the connection pool. When it is checked out again, if BeginTx is called on it again _without_ an isolation level, * then we fall into the mc.closed.IsSet() check in begin(), * so we return ErrBadConn, * so the driver kicks the broken connection out of the pool * (and transparently retries to get a new connection that isn't broken too). However, if BeginTx is called on the connection _with_ an isolation level, then we return a context canceled error from the SET TRANSACTION ISOLATION LEVEL call. That means the broken connection will stick around in the pool forever (or until it's checked out for some other operation that correctly returns ErrBadConn). The fix is to check for the connection being closed before executing SET TRANSACTION ISOLATION LEVEL. --- AUTHORS | 1 + connection.go | 4 ++++ driver_test.go | 19 +++++++++++++++++-- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 221f4a395..98cb1e66f 100644 --- a/AUTHORS +++ b/AUTHORS @@ -107,3 +107,4 @@ Multiplay Ltd. Percona LLC Pivotal Inc. Stripe Inc. +Zendesk Inc. diff --git a/connection.go b/connection.go index b07cd7651..6769e3ce1 100644 --- a/connection.go +++ b/connection.go @@ -489,6 +489,10 @@ func (mc *mysqlConn) Ping(ctx context.Context) (err error) { // BeginTx implements driver.ConnBeginTx interface func (mc *mysqlConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + if mc.closed.IsSet() { + return nil, driver.ErrBadConn + } + if err := mc.watchCancel(ctx); err != nil { return nil, err } diff --git a/driver_test.go b/driver_test.go index 8edd17c47..34b476ed3 100644 --- a/driver_test.go +++ b/driver_test.go @@ -2608,7 +2608,12 @@ func TestContextCancelBegin(t *testing.T) { runTests(t, dsn, func(dbt *DBTest) { dbt.mustExec("CREATE TABLE test (v INTEGER)") ctx, cancel := context.WithCancel(context.Background()) - tx, err := dbt.db.BeginTx(ctx, nil) + conn, err := dbt.db.Conn(ctx) + if err != nil { + dbt.Fatal(err) + } + defer conn.Close() + tx, err := conn.BeginTx(ctx, nil) if err != nil { dbt.Fatal(err) } @@ -2638,7 +2643,17 @@ func TestContextCancelBegin(t *testing.T) { dbt.Errorf("expected sql.ErrTxDone or context.Canceled, got %v", err) } - // Context is canceled, so cannot begin a transaction. + // The connection is now in an inoperable state - so performing other + // operations should fail with ErrBadConn + // Important to exercise isolation level too - it runs SET TRANSACTION ISOLATION + // LEVEL XXX first, which needs to return ErrBadConn if the connection's context + // is cancelled + _, err = conn.BeginTx(context.Background(), &sql.TxOptions{Isolation: sql.LevelReadCommitted}) + if err != driver.ErrBadConn { + dbt.Errorf("expected driver.ErrBadConn, got %v", err) + } + + // cannot begin a transaction (on a different conn) with a canceled context if _, err := dbt.db.BeginTx(ctx, nil); err != context.Canceled { dbt.Errorf("expected context.Canceled, got %v", err) } From f378f59f67100fe0079a06c4c5588a1f4df06227 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Mengu=C3=A9?= Date: Thu, 14 May 2020 01:43:51 +0200 Subject: [PATCH 57/93] Update travis: use Go 1.14 for testing (#1100) * Travis-CI: reverse order of Go version Reverse order of Go versions so the recent ones are tested first. * Travis-CI: add Go 1.14 * Travis-CI: move 'tip' just below the latest Go release * Travis-CI: upgrade Go to 1.14 to run tests --- .travis.yml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 506b4f4fb..74d9cd022 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,14 @@ language: go go: - - 1.10.x - - 1.11.x - - 1.12.x - - 1.13.x + # Keep the most recent production release at the top + - 1.14.x + # Go development version - master + # Older production releases + - 1.13.x + - 1.12.x + - 1.11.x + - 1.10.x before_install: - go get golang.org/x/tools/cmd/cover @@ -23,7 +27,7 @@ jobs: include: - env: DB=MYSQL8 dist: trusty - go: 1.10.x + go: 1.14.x services: - docker before_install: @@ -42,7 +46,7 @@ jobs: - env: DB=MYSQL57 dist: trusty - go: 1.10.x + go: 1.14.x services: - docker before_install: @@ -61,7 +65,7 @@ jobs: - env: DB=MARIA55 dist: trusty - go: 1.10.x + go: 1.14.x services: - docker before_install: @@ -80,7 +84,7 @@ jobs: - env: DB=MARIA10_1 dist: trusty - go: 1.10.x + go: 1.14.x services: - docker before_install: @@ -104,7 +108,7 @@ jobs: packages: - mysql update: true - go: 1.12.x + go: 1.14.x before_install: - go get golang.org/x/tools/cmd/cover - go get github.com/mattn/goveralls From 096feaaf8e9ffa009d7aa1c1b7eb57367b84d40c Mon Sep 17 00:00:00 2001 From: chanxuehong Date: Sun, 17 May 2020 23:48:53 +0800 Subject: [PATCH 58/93] performance improvement for parseDateTime (#1098) --- AUTHORS | 1 + utils.go | 14 ++++---------- utils_test.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/AUTHORS b/AUTHORS index 98cb1e66f..e24231d81 100644 --- a/AUTHORS +++ b/AUTHORS @@ -90,6 +90,7 @@ Vladimir Kovpak Xiangyu Hu Xiaobing Jiang Xiuming Chen +Xuehong Chan Zhenye Xie Zhixin Wen diff --git a/utils.go b/utils.go index 9552e80b5..154ecc337 100644 --- a/utils.go +++ b/utils.go @@ -113,20 +113,14 @@ func parseDateTime(str string, loc *time.Location) (t time.Time, err error) { if str == base[:len(str)] { return } - t, err = time.Parse(timeFormat[:len(str)], str) + if loc == time.UTC { + return time.Parse(timeFormat[:len(str)], str) + } + return time.ParseInLocation(timeFormat[:len(str)], str, loc) default: err = fmt.Errorf("invalid time string: %s", str) return } - - // Adjust location - if err == nil && loc != time.UTC { - y, mo, d := t.Date() - h, mi, s := t.Clock() - t, err = time.Date(y, mo, d, h, mi, s, t.Nanosecond(), loc), nil - } - - return } func parseBinaryDateTime(num uint64, data []byte, loc *time.Location) (driver.Value, error) { diff --git a/utils_test.go b/utils_test.go index 10a60c2d0..ab29cad78 100644 --- a/utils_test.go +++ b/utils_test.go @@ -14,6 +14,7 @@ import ( "database/sql/driver" "encoding/binary" "testing" + "time" ) func TestLengthEncodedInteger(t *testing.T) { @@ -291,3 +292,45 @@ func TestIsolationLevelMapping(t *testing.T) { t.Fatalf("Expected error to be %q, got %q", expectedErr, err) } } + +func TestParseDateTime(t *testing.T) { + // UTC loc + { + str := "2020-05-13 21:30:45" + t1, err := parseDateTime(str, time.UTC) + if err != nil { + t.Error(err) + return + } + t2 := time.Date(2020, 5, 13, + 21, 30, 45, 0, time.UTC) + if !t1.Equal(t2) { + t.Errorf("want equal, have: %v, want: %v", t1, t2) + return + } + } + // non-UTC loc + { + str := "2020-05-13 21:30:45" + loc := time.FixedZone("test", 8*60*60) + t1, err := parseDateTime(str, loc) + if err != nil { + t.Error(err) + return + } + t2 := time.Date(2020, 5, 13, + 21, 30, 45, 0, loc) + if !t1.Equal(t2) { + t.Errorf("want equal, have: %v, want: %v", t1, t2) + return + } + } +} + +func BenchmarkParseDateTime(b *testing.B) { + str := "2020-05-13 21:30:45" + loc := time.FixedZone("test", 8*60*60) + for i := 0; i < b.N; i++ { + _, _ = parseDateTime(str, loc) + } +} From 3f51e4ed70f7313fd12889ebc93fb42ab67d4941 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Mengu=C3=A9?= Date: Tue, 19 May 2020 14:52:07 +0200 Subject: [PATCH 59/93] On connect, set all variables in a single SET statement (#1099) When opening a connection, instead of iterating on all variables and calling "SET =" for each, we now use a single SET statement with all pairs using a comma as separator: SET =,=,... --- connection.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/connection.go b/connection.go index 6769e3ce1..0c280443d 100644 --- a/connection.go +++ b/connection.go @@ -47,9 +47,10 @@ type mysqlConn struct { // Handles parameters set in DSN after the connection is established func (mc *mysqlConn) handleParams() (err error) { + var params []string for param, val := range mc.cfg.Params { switch param { - // Charset + // Charset: character_set_connection, character_set_client, character_set_results case "charset": charsets := strings.Split(val, ",") for i := range charsets { @@ -63,12 +64,16 @@ func (mc *mysqlConn) handleParams() (err error) { return } - // System Vars + // Other system vars default: - err = mc.exec("SET " + param + "=" + val + "") - if err != nil { - return - } + params = append(params, param+"="+val) + } + } + + if len(params) > 0 { + err = mc.exec("SET " + strings.Join(params, ",")) + if err != nil { + return } } From 128a6737a20278b625f8de647998fec7b7a2dc3e Mon Sep 17 00:00:00 2001 From: AdamKorcz <44787359+AdamKorcz@users.noreply.github.com> Date: Thu, 21 May 2020 14:24:19 +0100 Subject: [PATCH 60/93] Add fuzzer (#1097) The fuzzer targets the `Open` function, which can be run locally as well as through oss-fuzz's infrastructure. --- fuzz.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 fuzz.go diff --git a/fuzz.go b/fuzz.go new file mode 100644 index 000000000..fa75adf6a --- /dev/null +++ b/fuzz.go @@ -0,0 +1,24 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package. +// +// Copyright 2020 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build gofuzz + +package mysql + +import ( + "database/sql" +) + +func Fuzz(data []byte) int { + db, err := sql.Open("mysql", string(data)) + if err != nil { + return 0 + } + db.Close() + return 1 +} From db7a5f2f6f8d399089e805f53a41014e988d365f Mon Sep 17 00:00:00 2001 From: Julien Schmidt Date: Mon, 25 May 2020 12:28:14 +0200 Subject: [PATCH 61/93] travis: do not run compile check in matrix builds (#1104) --- .travis.yml | 7 ++++++- .travis/{complie_check.sh => compile_check.sh} | 0 2 files changed, 6 insertions(+), 1 deletion(-) rename .travis/{complie_check.sh => compile_check.sh} (100%) diff --git a/.travis.yml b/.travis.yml index 74d9cd022..5639e4fd2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,6 +39,7 @@ jobs: - cp .travis/docker.cnf ~/.my.cnf - .travis/wait_mysql.sh before_script: + - export cross_compile=false - export MYSQL_TEST_USER=gotest - export MYSQL_TEST_PASS=secret - export MYSQL_TEST_ADDR=127.0.0.1:3307 @@ -58,6 +59,7 @@ jobs: - cp .travis/docker.cnf ~/.my.cnf - .travis/wait_mysql.sh before_script: + - export cross_compile=false - export MYSQL_TEST_USER=gotest - export MYSQL_TEST_PASS=secret - export MYSQL_TEST_ADDR=127.0.0.1:3307 @@ -77,6 +79,7 @@ jobs: - cp .travis/docker.cnf ~/.my.cnf - .travis/wait_mysql.sh before_script: + - export cross_compile=false - export MYSQL_TEST_USER=gotest - export MYSQL_TEST_PASS=secret - export MYSQL_TEST_ADDR=127.0.0.1:3307 @@ -96,6 +99,7 @@ jobs: - cp .travis/docker.cnf ~/.my.cnf - .travis/wait_mysql.sh before_script: + - export cross_compile=false - export MYSQL_TEST_USER=gotest - export MYSQL_TEST_PASS=secret - export MYSQL_TEST_ADDR=127.0.0.1:3307 @@ -113,6 +117,7 @@ jobs: - go get golang.org/x/tools/cmd/cover - go get github.com/mattn/goveralls before_script: + - export cross_compile=false - echo -e "[server]\ninnodb_log_file_size=256MB\ninnodb_buffer_pool_size=512MB\nmax_allowed_packet=16MB\nlocal_infile=1" >> /usr/local/etc/my.cnf - mysql.server start - mysql -uroot -e 'CREATE USER gotest IDENTIFIED BY "secret"' @@ -127,6 +132,6 @@ script: - go test -v -covermode=count -coverprofile=coverage.out - go vet ./... - .travis/gofmt.sh - - .travis/complie_check.sh + - if [ "$cross_compile" != "false" ]; then .travis/compile_check.sh; fi after_script: - $HOME/gopath/bin/goveralls -coverprofile=coverage.out -service=travis-ci diff --git a/.travis/complie_check.sh b/.travis/compile_check.sh similarity index 100% rename from .travis/complie_check.sh rename to .travis/compile_check.sh From 8c3a2d9049b4fc1a914673c22bdef3741b8bea7e Mon Sep 17 00:00:00 2001 From: Julien Schmidt Date: Mon, 25 May 2020 12:28:40 +0200 Subject: [PATCH 62/93] travis: update image versions (#1103) * travis: update image versions * travis: use mysql client inside the docker container for polling * travis: force mysql client to connect via tcp * travis: use root pass in mysql client * travis: use empty root password --- .travis.yml | 18 +++++++++--------- .travis/wait_mysql.sh | 8 +++++++- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5639e4fd2..21638db0e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ jobs: include: - env: DB=MYSQL8 - dist: trusty + dist: xenial go: 1.14.x services: - docker @@ -34,7 +34,7 @@ jobs: - go get golang.org/x/tools/cmd/cover - go get github.com/mattn/goveralls - docker pull mysql:8.0 - - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ROOT_PASSWORD=verysecret + - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ALLOW_EMPTY_PASSWORD=yes mysql:8.0 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 - cp .travis/docker.cnf ~/.my.cnf - .travis/wait_mysql.sh @@ -46,7 +46,7 @@ jobs: - export MYSQL_TEST_CONCURRENT=1 - env: DB=MYSQL57 - dist: trusty + dist: xenial go: 1.14.x services: - docker @@ -54,7 +54,7 @@ jobs: - go get golang.org/x/tools/cmd/cover - go get github.com/mattn/goveralls - docker pull mysql:5.7 - - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ROOT_PASSWORD=verysecret + - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ALLOW_EMPTY_PASSWORD=yes mysql:5.7 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 - cp .travis/docker.cnf ~/.my.cnf - .travis/wait_mysql.sh @@ -66,7 +66,7 @@ jobs: - export MYSQL_TEST_CONCURRENT=1 - env: DB=MARIA55 - dist: trusty + dist: xenial go: 1.14.x services: - docker @@ -74,7 +74,7 @@ jobs: - go get golang.org/x/tools/cmd/cover - go get github.com/mattn/goveralls - docker pull mariadb:5.5 - - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ROOT_PASSWORD=verysecret + - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ALLOW_EMPTY_PASSWORD=yes mariadb:5.5 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 - cp .travis/docker.cnf ~/.my.cnf - .travis/wait_mysql.sh @@ -86,7 +86,7 @@ jobs: - export MYSQL_TEST_CONCURRENT=1 - env: DB=MARIA10_1 - dist: trusty + dist: xenial go: 1.14.x services: - docker @@ -94,7 +94,7 @@ jobs: - go get golang.org/x/tools/cmd/cover - go get github.com/mattn/goveralls - docker pull mariadb:10.1 - - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ROOT_PASSWORD=verysecret + - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ALLOW_EMPTY_PASSWORD=yes mariadb:10.1 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 - cp .travis/docker.cnf ~/.my.cnf - .travis/wait_mysql.sh @@ -106,7 +106,7 @@ jobs: - export MYSQL_TEST_CONCURRENT=1 - os: osx - osx_image: xcode10.1 + osx_image: xcode11.4 addons: homebrew: packages: diff --git a/.travis/wait_mysql.sh b/.travis/wait_mysql.sh index e87993e57..fa2054ff1 100755 --- a/.travis/wait_mysql.sh +++ b/.travis/wait_mysql.sh @@ -1,7 +1,13 @@ #!/bin/sh + +# use the mysql client inside the docker container if docker is running +[ "$(docker inspect -f '{{.State.Running}}' mysqld 2>/dev/null)" = "true" ] && mysql() { + docker exec mysqld mysql "${@}" +} + while : do - if mysql -e 'select version()' 2>&1 | grep 'version()\|ERROR 2059 (HY000):'; then + if mysql --protocol=tcp -e 'select version()'; then break fi sleep 3 From 26060e1824b523760ce8c6a6794b34b20c3f220d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Mengu=C3=A9?= Date: Fri, 29 May 2020 10:48:22 +0200 Subject: [PATCH 63/93] connect: reduce allocations when building SET command (#1111) * connect: reduce allocations when building SET command * handleParams: use strings.Builder instead of direct []byte --- connection.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/connection.go b/connection.go index 0c280443d..d1d8b29fe 100644 --- a/connection.go +++ b/connection.go @@ -47,7 +47,7 @@ type mysqlConn struct { // Handles parameters set in DSN after the connection is established func (mc *mysqlConn) handleParams() (err error) { - var params []string + var cmdSet strings.Builder for param, val := range mc.cfg.Params { switch param { // Charset: character_set_connection, character_set_client, character_set_results @@ -64,14 +64,23 @@ func (mc *mysqlConn) handleParams() (err error) { return } - // Other system vars + // Other system vars accumulated in a single SET command default: - params = append(params, param+"="+val) + if cmdSet.Len() == 0 { + // Heuristic: 29 chars for each other key=value to reduce reallocations + cmdSet.Grow(4 + len(param) + 1 + len(val) + 30*(len(mc.cfg.Params)-1)) + cmdSet.WriteString("SET ") + } else { + cmdSet.WriteByte(',') + } + cmdSet.WriteString(param) + cmdSet.WriteByte('=') + cmdSet.WriteString(val) } } - if len(params) > 0 { - err = mc.exec("SET " + strings.Join(params, ",")) + if cmdSet.Len() > 0 { + err = mc.exec(cmdSet.String()) if err != nil { return } From d2e52fca0b6562a2e2cb8252fbcdbab6a6e8e2d6 Mon Sep 17 00:00:00 2001 From: Caine Jette Date: Fri, 29 May 2020 01:50:06 -0700 Subject: [PATCH 64/93] Replace whitelist/blacklist terminology with allowlist/denylist (#1116) * Replace whitelist/blacklist terminology with allowlist/denylist * Add myself to AUTHORS * PR feedback * Denylist --> denied * Update denied --> rejected --- AUTHORS | 1 + README.md | 8 ++++---- collations.go | 2 +- dsn.go | 2 +- infile.go | 4 ++-- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/AUTHORS b/AUTHORS index e24231d81..acc842daf 100644 --- a/AUTHORS +++ b/AUTHORS @@ -21,6 +21,7 @@ Arne Hormann Ariel Mashraki Asta Xie Bulat Gaifullin +Caine Jette Carlos Nieto Chris Moos Craig Wilson diff --git a/README.md b/README.md index 6ac81d1e4..1eeb9794a 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) pac * Supports queries larger than 16MB * Full [`sql.RawBytes`](https://golang.org/pkg/database/sql/#RawBytes) support. * Intelligent `LONG DATA` handling in prepared statements - * Secure `LOAD DATA LOCAL INFILE` support with file Whitelisting and `io.Reader` support + * Secure `LOAD DATA LOCAL INFILE` support with file allowlisting and `io.Reader` support * Optional `time.Time` parsing * Optional placeholder interpolation @@ -122,7 +122,7 @@ Valid Values: true, false Default: false ``` -`allowAllFiles=true` disables the file Whitelist for `LOAD DATA LOCAL INFILE` and allows *all* files. +`allowAllFiles=true` disables the file allowlist for `LOAD DATA LOCAL INFILE` and allows *all* files. [*Might be insecure!*](http://dev.mysql.com/doc/refman/5.7/en/load-data-local.html) ##### `allowCleartextPasswords` @@ -230,7 +230,7 @@ Default: false If `interpolateParams` is true, placeholders (`?`) in calls to `db.Query()` and `db.Exec()` are interpolated into a single query string with given parameters. This reduces the number of roundtrips, since the driver has to prepare a statement, execute it with given parameters and close the statement again with `interpolateParams=false`. -*This can not be used together with the multibyte encodings BIG5, CP932, GB2312, GBK or SJIS. These are blacklisted as they may [introduce a SQL injection vulnerability](http://stackoverflow.com/a/12118602/3430118)!* +*This can not be used together with the multibyte encodings BIG5, CP932, GB2312, GBK or SJIS. These are rejected as they may [introduce a SQL injection vulnerability](http://stackoverflow.com/a/12118602/3430118)!* ##### `loc` @@ -445,7 +445,7 @@ For this feature you need direct access to the package. Therefore you must chang import "github.com/go-sql-driver/mysql" ``` -Files must be whitelisted by registering them with `mysql.RegisterLocalFile(filepath)` (recommended) or the Whitelist check must be deactivated by using the DSN parameter `allowAllFiles=true` ([*Might be insecure!*](http://dev.mysql.com/doc/refman/5.7/en/load-data-local.html)). +Files must be explicitly allowed by registering them with `mysql.RegisterLocalFile(filepath)` (recommended) or the allowlist check must be deactivated by using the DSN parameter `allowAllFiles=true` ([*Might be insecure!*](http://dev.mysql.com/doc/refman/5.7/en/load-data-local.html)). To use a `io.Reader` a handler function must be registered with `mysql.RegisterReaderHandler(name, handler)` which returns a `io.Reader` or `io.ReadCloser`. The Reader is available with the filepath `Reader::` then. Choose different names for different handlers and `DeregisterReaderHandler` when you don't need it anymore. diff --git a/collations.go b/collations.go index 8d2b55676..326a9f7fa 100644 --- a/collations.go +++ b/collations.go @@ -247,7 +247,7 @@ var collations = map[string]byte{ "utf8mb4_0900_ai_ci": 255, } -// A blacklist of collations which is unsafe to interpolate parameters. +// A denylist of collations which is unsafe to interpolate parameters. // These multibyte encodings may contains 0x5c (`\`) in their trailing bytes. var unsafeCollations = map[string]bool{ "big5_chinese_ci": true, diff --git a/dsn.go b/dsn.go index 75c8c2489..93f3548cb 100644 --- a/dsn.go +++ b/dsn.go @@ -375,7 +375,7 @@ func parseDSNParams(cfg *Config, params string) (err error) { // cfg params switch value := param[1]; param[0] { - // Disable INFILE whitelist / enable all files + // Disable INFILE allowlist / enable all files case "allowAllFiles": var isBool bool cfg.AllowAllFiles, isBool = readBool(value) diff --git a/infile.go b/infile.go index 273cb0ba5..60effdfc2 100644 --- a/infile.go +++ b/infile.go @@ -23,7 +23,7 @@ var ( readerRegisterLock sync.RWMutex ) -// RegisterLocalFile adds the given file to the file whitelist, +// RegisterLocalFile adds the given file to the file allowlist, // so that it can be used by "LOAD DATA LOCAL INFILE ". // Alternatively you can allow the use of all local files with // the DSN parameter 'allowAllFiles=true' @@ -45,7 +45,7 @@ func RegisterLocalFile(filePath string) { fileRegisterLock.Unlock() } -// DeregisterLocalFile removes the given filepath from the whitelist. +// DeregisterLocalFile removes the given filepath from the allowlist. func DeregisterLocalFile(filePath string) { fileRegisterLock.Lock() delete(fileRegister, strings.Trim(filePath, `"`)) From 12508c83901b1a418e3dea51850f956f32506f4e Mon Sep 17 00:00:00 2001 From: Kei Kamikawa Date: Sun, 31 May 2020 19:04:19 +0900 Subject: [PATCH 65/93] utils: parse using byteslice in parseDateTime (#1113) * fixed the way of parsing datetime when byte slice string The benchmark results $ go test -benchmem . -bench "^BenchmarkParseByte" goos: darwin goarch: amd64 pkg: github.com/go-sql-driver/mysql BenchmarkParseByteDateTime-4 12023173 104 ns/op 0 B/op 0 allocs/op BenchmarkParseByteDateTimeStringCast-4 3394239 355 ns/op 32 B/op 1 allocs/op * added line to AUTHORS file * fixed error handling * fixed nanosec digits * added more tests for error * renamed parseByteDateTime to parseDateTime * reverted base null time * Update utils.go Co-authored-by: Inada Naoki * Update utils.go Co-authored-by: Inada Naoki * Update utils.go Co-authored-by: Inada Naoki * removed deprecatedParseDateTime from test Co-authored-by: Inada Naoki --- AUTHORS | 1 + nulltime.go | 4 +- packets.go | 2 +- utils.go | 135 +++++++++++++++++++++++++++++++++++++--- utils_test.go | 167 ++++++++++++++++++++++++++++++++++++++++---------- 5 files changed, 263 insertions(+), 46 deletions(-) diff --git a/AUTHORS b/AUTHORS index acc842daf..8ba3db6b3 100644 --- a/AUTHORS +++ b/AUTHORS @@ -56,6 +56,7 @@ Julien Schmidt Justin Li Justin Nuß Kamil Dziedzic +Kei Kamikawa Kevin Malachowski Kieron Woodhouse Lennart Rudolph diff --git a/nulltime.go b/nulltime.go index afa8a89e9..651723a96 100644 --- a/nulltime.go +++ b/nulltime.go @@ -28,11 +28,11 @@ func (nt *NullTime) Scan(value interface{}) (err error) { nt.Time, nt.Valid = v, true return case []byte: - nt.Time, err = parseDateTime(string(v), time.UTC) + nt.Time, err = parseDateTime(v, time.UTC) nt.Valid = (err == nil) return case string: - nt.Time, err = parseDateTime(v, time.UTC) + nt.Time, err = parseDateTime([]byte(v), time.UTC) nt.Valid = (err == nil) return } diff --git a/packets.go b/packets.go index 5cbd53298..8e2f5e76f 100644 --- a/packets.go +++ b/packets.go @@ -778,7 +778,7 @@ func (rows *textRows) readRow(dest []driver.Value) error { case fieldTypeTimestamp, fieldTypeDateTime, fieldTypeDate, fieldTypeNewDate: dest[i], err = parseDateTime( - string(dest[i].([]byte)), + dest[i].([]byte), mc.cfg.Loc, ) if err == nil { diff --git a/utils.go b/utils.go index 154ecc337..9dd3679c6 100644 --- a/utils.go +++ b/utils.go @@ -106,21 +106,136 @@ func readBool(input string) (value bool, valid bool) { * Time related utils * ******************************************************************************/ -func parseDateTime(str string, loc *time.Location) (t time.Time, err error) { - base := "0000-00-00 00:00:00.0000000" - switch len(str) { +func parseDateTime(b []byte, loc *time.Location) (time.Time, error) { + const base = "0000-00-00 00:00:00.000000" + switch len(b) { case 10, 19, 21, 22, 23, 24, 25, 26: // up to "YYYY-MM-DD HH:MM:SS.MMMMMM" - if str == base[:len(str)] { - return + if string(b) == base[:len(b)] { + return time.Time{}, nil } - if loc == time.UTC { - return time.Parse(timeFormat[:len(str)], str) + + year, err := parseByteYear(b) + if err != nil { + return time.Time{}, err + } + if year <= 0 { + year = 1 + } + + if b[4] != '-' { + return time.Time{}, fmt.Errorf("bad value for field: `%c`", b[4]) + } + + m, err := parseByte2Digits(b[5], b[6]) + if err != nil { + return time.Time{}, err + } + if m <= 0 { + m = 1 + } + month := time.Month(m) + + if b[7] != '-' { + return time.Time{}, fmt.Errorf("bad value for field: `%c`", b[7]) + } + + day, err := parseByte2Digits(b[8], b[9]) + if err != nil { + return time.Time{}, err + } + if day <= 0 { + day = 1 + } + if len(b) == 10 { + return time.Date(year, month, day, 0, 0, 0, 0, loc), nil + } + + if b[10] != ' ' { + return time.Time{}, fmt.Errorf("bad value for field: `%c`", b[10]) + } + + hour, err := parseByte2Digits(b[11], b[12]) + if err != nil { + return time.Time{}, err + } + if b[13] != ':' { + return time.Time{}, fmt.Errorf("bad value for field: `%c`", b[13]) + } + + min, err := parseByte2Digits(b[14], b[15]) + if err != nil { + return time.Time{}, err + } + if b[16] != ':' { + return time.Time{}, fmt.Errorf("bad value for field: `%c`", b[16]) } - return time.ParseInLocation(timeFormat[:len(str)], str, loc) + + sec, err := parseByte2Digits(b[17], b[18]) + if err != nil { + return time.Time{}, err + } + if len(b) == 19 { + return time.Date(year, month, day, hour, min, sec, 0, loc), nil + } + + if b[19] != '.' { + return time.Time{}, fmt.Errorf("bad value for field: `%c`", b[19]) + } + nsec, err := parseByteNanoSec(b[20:]) + if err != nil { + return time.Time{}, err + } + return time.Date(year, month, day, hour, min, sec, nsec, loc), nil default: - err = fmt.Errorf("invalid time string: %s", str) - return + return time.Time{}, fmt.Errorf("invalid time bytes: %s", b) + } +} + +func parseByteYear(b []byte) (int, error) { + year, n := 0, 1000 + for i := 0; i < 4; i++ { + v, err := bToi(b[i]) + if err != nil { + return 0, err + } + year += v * n + n = n / 10 + } + return year, nil +} + +func parseByte2Digits(b1, b2 byte) (int, error) { + d1, err := bToi(b1) + if err != nil { + return 0, err + } + d2, err := bToi(b2) + if err != nil { + return 0, err + } + return d1*10 + d2, nil +} + +func parseByteNanoSec(b []byte) (int, error) { + ns, digit := 0, 100000 // max is 6-digits + for i := 0; i < len(b); i++ { + v, err := bToi(b[i]) + if err != nil { + return 0, err + } + ns += v * digit + digit /= 10 + } + // nanoseconds has 10-digits. (needs to scale digits) + // 10 - 6 = 4, so we have to multiple 1000. + return ns * 1000, nil +} + +func bToi(b byte) (int, error) { + if b < '0' || b > '9' { + return 0, errors.New("not [0-9]") } + return int(b - '0'), nil } func parseBinaryDateTime(num uint64, data []byte, loc *time.Location) (driver.Value, error) { diff --git a/utils_test.go b/utils_test.go index ab29cad78..114f4b3da 100644 --- a/utils_test.go +++ b/utils_test.go @@ -294,43 +294,144 @@ func TestIsolationLevelMapping(t *testing.T) { } func TestParseDateTime(t *testing.T) { - // UTC loc - { - str := "2020-05-13 21:30:45" - t1, err := parseDateTime(str, time.UTC) - if err != nil { - t.Error(err) - return - } - t2 := time.Date(2020, 5, 13, - 21, 30, 45, 0, time.UTC) - if !t1.Equal(t2) { - t.Errorf("want equal, have: %v, want: %v", t1, t2) - return - } + cases := []struct { + name string + str string + }{ + { + name: "parse date", + str: "2020-05-13", + }, + { + name: "parse null date", + str: sDate0, + }, + { + name: "parse datetime", + str: "2020-05-13 21:30:45", + }, + { + name: "parse null datetime", + str: sDateTime0, + }, + { + name: "parse datetime nanosec 1-digit", + str: "2020-05-25 23:22:01.1", + }, + { + name: "parse datetime nanosec 2-digits", + str: "2020-05-25 23:22:01.15", + }, + { + name: "parse datetime nanosec 3-digits", + str: "2020-05-25 23:22:01.159", + }, + { + name: "parse datetime nanosec 4-digits", + str: "2020-05-25 23:22:01.1594", + }, + { + name: "parse datetime nanosec 5-digits", + str: "2020-05-25 23:22:01.15949", + }, + { + name: "parse datetime nanosec 6-digits", + str: "2020-05-25 23:22:01.159491", + }, } - // non-UTC loc - { - str := "2020-05-13 21:30:45" - loc := time.FixedZone("test", 8*60*60) - t1, err := parseDateTime(str, loc) - if err != nil { - t.Error(err) - return - } - t2 := time.Date(2020, 5, 13, - 21, 30, 45, 0, loc) - if !t1.Equal(t2) { - t.Errorf("want equal, have: %v, want: %v", t1, t2) - return + + for _, loc := range []*time.Location{ + time.UTC, + time.FixedZone("test", 8*60*60), + } { + for _, cc := range cases { + t.Run(cc.name+"-"+loc.String(), func(t *testing.T) { + var want time.Time + if cc.str != sDate0 && cc.str != sDateTime0 { + var err error + want, err = time.ParseInLocation(timeFormat[:len(cc.str)], cc.str, loc) + if err != nil { + t.Fatal(err) + } + } + got, err := parseDateTime([]byte(cc.str), loc) + if err != nil { + t.Fatal(err) + } + + if !want.Equal(got) { + t.Fatalf("want: %v, but got %v", want, got) + } + }) } } } -func BenchmarkParseDateTime(b *testing.B) { - str := "2020-05-13 21:30:45" - loc := time.FixedZone("test", 8*60*60) - for i := 0; i < b.N; i++ { - _, _ = parseDateTime(str, loc) +func TestParseDateTimeFail(t *testing.T) { + cases := []struct { + name string + str string + wantErr string + }{ + { + name: "parse invalid time", + str: "hello", + wantErr: "invalid time bytes: hello", + }, + { + name: "parse year", + str: "000!-00-00 00:00:00.000000", + wantErr: "not [0-9]", + }, + { + name: "parse month", + str: "0000-!0-00 00:00:00.000000", + wantErr: "not [0-9]", + }, + { + name: `parse "-" after parsed year`, + str: "0000:00-00 00:00:00.000000", + wantErr: "bad value for field: `:`", + }, + { + name: `parse "-" after parsed month`, + str: "0000-00:00 00:00:00.000000", + wantErr: "bad value for field: `:`", + }, + { + name: `parse " " after parsed date`, + str: "0000-00-00+00:00:00.000000", + wantErr: "bad value for field: `+`", + }, + { + name: `parse ":" after parsed date`, + str: "0000-00-00 00-00:00.000000", + wantErr: "bad value for field: `-`", + }, + { + name: `parse ":" after parsed hour`, + str: "0000-00-00 00:00-00.000000", + wantErr: "bad value for field: `-`", + }, + { + name: `parse "." after parsed sec`, + str: "0000-00-00 00:00:00?000000", + wantErr: "bad value for field: `?`", + }, + } + + for _, cc := range cases { + t.Run(cc.name, func(t *testing.T) { + got, err := parseDateTime([]byte(cc.str), time.UTC) + if err == nil { + t.Fatal("want error") + } + if cc.wantErr != err.Error() { + t.Fatalf("want `%s`, but got `%s`", cc.wantErr, err) + } + if !got.IsZero() { + t.Fatal("want zero time") + } + }) } } From 73dc904a9ece5c074295a77abb7a135797a351bf Mon Sep 17 00:00:00 2001 From: nozawana44 <62693119+nozawana44@users.noreply.github.com> Date: Mon, 20 Jul 2020 16:11:43 +0900 Subject: [PATCH 66/93] README: replace tx_isolation with transaction_isolation (#1077) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1eeb9794a..505d2a7e1 100644 --- a/README.md +++ b/README.md @@ -376,7 +376,7 @@ Rules: Examples: * `autocommit=1`: `SET autocommit=1` * [`time_zone=%27Europe%2FParis%27`](https://dev.mysql.com/doc/refman/5.5/en/time-zone-support.html): `SET time_zone='Europe/Paris'` - * [`tx_isolation=%27REPEATABLE-READ%27`](https://dev.mysql.com/doc/refman/5.5/en/server-system-variables.html#sysvar_tx_isolation): `SET tx_isolation='REPEATABLE-READ'` + * [`transaction_isolation=%27REPEATABLE-READ%27`](https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_transaction_isolation): `SET transaction_isolation='REPEATABLE-READ'` #### Examples From 076901a81ee91628a5c11d46584053047c10d1d5 Mon Sep 17 00:00:00 2001 From: IKEDA Sho Date: Thu, 13 Aug 2020 13:40:09 +0900 Subject: [PATCH 67/93] Support returning uint64 from Valuer in ConvertValue (#1143) https://golang.org/pkg/database/sql/driver/#Value says: > Value is a value that drivers must be able to handle. It is either nil, a type handled by a database driver's NamedValueChecker interface, or an instance of one of these types: --- AUTHORS | 1 + statement.go | 12 +++++++++--- statement_test.go | 10 ++++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/AUTHORS b/AUTHORS index 8ba3db6b3..ccf2f466c 100644 --- a/AUTHORS +++ b/AUTHORS @@ -79,6 +79,7 @@ Reed Allman Richard Wilkes Robert Russell Runrioter Wung +Sho Ikeda Shuode Li Simon J Mudd Soroush Pour diff --git a/statement.go b/statement.go index d3e68112f..18a3ae498 100644 --- a/statement.go +++ b/statement.go @@ -154,10 +154,16 @@ func (c converter) ConvertValue(v interface{}) (driver.Value, error) { if err != nil { return nil, err } - if !driver.IsValue(sv) { - return nil, fmt.Errorf("non-Value type %T returned from Value", sv) + if driver.IsValue(sv) { + return sv, nil } - return sv, nil + // A value returend from the Valuer interface can be "a type handled by + // a database driver's NamedValueChecker interface" so we should accept + // uint64 here as well. + if u, ok := sv.(uint64); ok { + return u, nil + } + return nil, fmt.Errorf("non-Value type %T returned from Value", sv) } rv := reflect.ValueOf(v) switch rv.Kind() { diff --git a/statement_test.go b/statement_test.go index 2cc022bf5..ac6b92de9 100644 --- a/statement_test.go +++ b/statement_test.go @@ -10,6 +10,7 @@ package mysql import ( "bytes" + "database/sql/driver" "encoding/json" "testing" ) @@ -96,6 +97,14 @@ func TestConvertSignedIntegers(t *testing.T) { } } +type myUint64 struct { + value uint64 +} + +func (u myUint64) Value() (driver.Value, error) { + return u.value, nil +} + func TestConvertUnsignedIntegers(t *testing.T) { values := []interface{}{ uint8(42), @@ -103,6 +112,7 @@ func TestConvertUnsignedIntegers(t *testing.T) { uint32(42), uint64(42), uint(42), + myUint64{uint64(42)}, } for _, value := range values { From 3b935426341bc5d229eafd936e4f4240da027ccd Mon Sep 17 00:00:00 2001 From: chanxuehong Date: Thu, 13 Aug 2020 12:43:20 +0800 Subject: [PATCH 68/93] performance improvement for time format (#1118) --- connection.go | 42 +++--------------------------- packets.go | 5 +++- utils.go | 49 +++++++++++++++++++++++++++++++++++ utils_test.go | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 129 insertions(+), 39 deletions(-) diff --git a/connection.go b/connection.go index d1d8b29fe..90aec6439 100644 --- a/connection.go +++ b/connection.go @@ -245,44 +245,10 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin if v.IsZero() { buf = append(buf, "'0000-00-00'"...) } else { - v := v.In(mc.cfg.Loc) - v = v.Add(time.Nanosecond * 500) // To round under microsecond - year := v.Year() - year100 := year / 100 - year1 := year % 100 - month := v.Month() - day := v.Day() - hour := v.Hour() - minute := v.Minute() - second := v.Second() - micro := v.Nanosecond() / 1000 - - buf = append(buf, []byte{ - '\'', - digits10[year100], digits01[year100], - digits10[year1], digits01[year1], - '-', - digits10[month], digits01[month], - '-', - digits10[day], digits01[day], - ' ', - digits10[hour], digits01[hour], - ':', - digits10[minute], digits01[minute], - ':', - digits10[second], digits01[second], - }...) - - if micro != 0 { - micro10000 := micro / 10000 - micro100 := micro / 100 % 100 - micro1 := micro % 100 - buf = append(buf, []byte{ - '.', - digits10[micro10000], digits01[micro10000], - digits10[micro100], digits01[micro100], - digits10[micro1], digits01[micro1], - }...) + buf = append(buf, '\'') + buf, err = appendDateTime(buf, v.In(mc.cfg.Loc)) + if err != nil { + return "", err } buf = append(buf, '\'') } diff --git a/packets.go b/packets.go index 8e2f5e76f..6664e5ae5 100644 --- a/packets.go +++ b/packets.go @@ -1116,7 +1116,10 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { if v.IsZero() { b = append(b, "0000-00-00"...) } else { - b = v.In(mc.cfg.Loc).AppendFormat(b, timeFormat) + b, err = appendDateTime(b, v.In(mc.cfg.Loc)) + if err != nil { + return err + } } paramValues = appendLengthEncodedInteger(paramValues, diff --git a/utils.go b/utils.go index 9dd3679c6..b0c6e9ca3 100644 --- a/utils.go +++ b/utils.go @@ -276,6 +276,55 @@ func parseBinaryDateTime(num uint64, data []byte, loc *time.Location) (driver.Va return nil, fmt.Errorf("invalid DATETIME packet length %d", num) } +func appendDateTime(buf []byte, t time.Time) ([]byte, error) { + nsec := t.Nanosecond() + // to round under microsecond + if nsec%1000 >= 500 { // save half of time.Time.Add calls + t = t.Add(500 * time.Nanosecond) + nsec = t.Nanosecond() + } + year, month, day := t.Date() + hour, min, sec := t.Clock() + micro := nsec / 1000 + + if year < 1 || year > 9999 { + return buf, errors.New("year is not in the range [1, 9999]: " + strconv.Itoa(year)) // use errors.New instead of fmt.Errorf to avoid year escape to heap + } + year100 := year / 100 + year1 := year % 100 + + var localBuf [26]byte // does not escape + localBuf[0], localBuf[1], localBuf[2], localBuf[3] = digits10[year100], digits01[year100], digits10[year1], digits01[year1] + localBuf[4] = '-' + localBuf[5], localBuf[6] = digits10[month], digits01[month] + localBuf[7] = '-' + localBuf[8], localBuf[9] = digits10[day], digits01[day] + + if hour == 0 && min == 0 && sec == 0 && micro == 0 { + return append(buf, localBuf[:10]...), nil + } + + localBuf[10] = ' ' + localBuf[11], localBuf[12] = digits10[hour], digits01[hour] + localBuf[13] = ':' + localBuf[14], localBuf[15] = digits10[min], digits01[min] + localBuf[16] = ':' + localBuf[17], localBuf[18] = digits10[sec], digits01[sec] + + if micro == 0 { + return append(buf, localBuf[:19]...), nil + } + + micro10000 := micro / 10000 + micro100 := (micro / 100) % 100 + micro1 := micro % 100 + localBuf[19] = '.' + localBuf[20], localBuf[21], localBuf[22], localBuf[23], localBuf[24], localBuf[25] = + digits10[micro10000], digits01[micro10000], digits10[micro100], digits01[micro100], digits10[micro1], digits01[micro1] + + return append(buf, localBuf[:]...), nil +} + // zeroDateTime is used in formatBinaryDateTime to avoid an allocation // if the DATE or DATETIME has the zero value. // It must never be changed. diff --git a/utils_test.go b/utils_test.go index 114f4b3da..e3619e7a7 100644 --- a/utils_test.go +++ b/utils_test.go @@ -293,6 +293,78 @@ func TestIsolationLevelMapping(t *testing.T) { } } +func TestAppendDateTime(t *testing.T) { + tests := []struct { + t time.Time + str string + }{ + { + t: time.Date(2020, 05, 30, 0, 0, 0, 0, time.UTC), + str: "2020-05-30", + }, + { + t: time.Date(2020, 05, 30, 22, 0, 0, 0, time.UTC), + str: "2020-05-30 22:00:00", + }, + { + t: time.Date(2020, 05, 30, 22, 33, 0, 0, time.UTC), + str: "2020-05-30 22:33:00", + }, + { + t: time.Date(2020, 05, 30, 22, 33, 44, 0, time.UTC), + str: "2020-05-30 22:33:44", + }, + { + t: time.Date(2020, 05, 30, 22, 33, 44, 550000000, time.UTC), + str: "2020-05-30 22:33:44.550000", + }, + { + t: time.Date(2020, 05, 30, 22, 33, 44, 550000499, time.UTC), + str: "2020-05-30 22:33:44.550000", + }, + { + t: time.Date(2020, 05, 30, 22, 33, 44, 550000500, time.UTC), + str: "2020-05-30 22:33:44.550001", + }, + { + t: time.Date(2020, 05, 30, 22, 33, 44, 550000567, time.UTC), + str: "2020-05-30 22:33:44.550001", + }, + { + t: time.Date(2020, 05, 30, 22, 33, 44, 999999567, time.UTC), + str: "2020-05-30 22:33:45", + }, + } + for _, v := range tests { + buf := make([]byte, 0, 32) + buf, _ = appendDateTime(buf, v.t) + if str := string(buf); str != v.str { + t.Errorf("appendDateTime(%v), have: %s, want: %s", v.t, str, v.str) + return + } + } + + // year out of range + { + v := time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC) + buf := make([]byte, 0, 32) + _, err := appendDateTime(buf, v) + if err == nil { + t.Error("want an error") + return + } + } + { + v := time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC) + buf := make([]byte, 0, 32) + _, err := appendDateTime(buf, v) + if err == nil { + t.Error("want an error") + return + } + } +} + func TestParseDateTime(t *testing.T) { cases := []struct { name string From f26d659cdb79eef0c396c764896dfaaa2fe93991 Mon Sep 17 00:00:00 2001 From: Ichinose Shogo Date: Thu, 13 Aug 2020 13:52:13 +0900 Subject: [PATCH 69/93] Add Go 1.15 to the build matrix (#1146) --- .travis.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 21638db0e..2926bdd11 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,11 @@ language: go go: # Keep the most recent production release at the top - - 1.14.x + - 1.15.x # Go development version - master # Older production releases + - 1.14.x - 1.13.x - 1.12.x - 1.11.x @@ -22,12 +23,12 @@ before_script: jobs: allow_failures: - - go: master + - go: master include: - env: DB=MYSQL8 dist: xenial - go: 1.14.x + go: 1.15.x services: - docker before_install: @@ -47,7 +48,7 @@ jobs: - env: DB=MYSQL57 dist: xenial - go: 1.14.x + go: 1.15.x services: - docker before_install: @@ -67,7 +68,7 @@ jobs: - env: DB=MARIA55 dist: xenial - go: 1.14.x + go: 1.15.x services: - docker before_install: @@ -87,7 +88,7 @@ jobs: - env: DB=MARIA10_1 dist: xenial - go: 1.14.x + go: 1.15.x services: - docker before_install: @@ -112,7 +113,7 @@ jobs: packages: - mysql update: true - go: 1.14.x + go: 1.15.x before_install: - go get golang.org/x/tools/cmd/cover - go get github.com/mattn/goveralls From b5b0ea55e9c131dcc4a3d5ccb70f06be52f72c9e Mon Sep 17 00:00:00 2001 From: Ichinose Shogo Date: Sun, 16 Aug 2020 11:32:30 +0900 Subject: [PATCH 70/93] Add deprecated message to NullTime (#1144) * Add deprecated message to NullTime * remove NullTime from README --- README.md | 2 -- nulltime_go113.go | 4 ++++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 505d2a7e1..834cf10f5 100644 --- a/README.md +++ b/README.md @@ -459,8 +459,6 @@ However, many want to scan MySQL `DATE` and `DATETIME` values into `time.Time` v **Caution:** As of Go 1.1, this makes `time.Time` the only variable type you can scan `DATE` and `DATETIME` values into. This breaks for example [`sql.RawBytes` support](https://github.com/go-sql-driver/mysql/wiki/Examples#rawbytes). -Alternatively you can use the [`NullTime`](https://godoc.org/github.com/go-sql-driver/mysql#NullTime) type as the scan destination, which works with both `time.Time` and `string` / `[]byte`. - ### Unicode support Since version 1.5 Go-MySQL-Driver automatically uses the collation ` utf8mb4_general_ci` by default. diff --git a/nulltime_go113.go b/nulltime_go113.go index c392594dd..cb9bcae4a 100644 --- a/nulltime_go113.go +++ b/nulltime_go113.go @@ -28,4 +28,8 @@ import ( // } // // This NullTime implementation is not driver-specific +// +// Deprecated: NullTime doesn't honor the loc DSN parameter. +// NullTime.Scan interprets a time as UTC, not the loc DSN parameter. +// Use sql.NullTime instead. type NullTime sql.NullTime From 84241ad3f47266ef3ee7cdbc13975c48ce66fa83 Mon Sep 17 00:00:00 2001 From: Ichinose Shogo Date: Sun, 16 Aug 2020 11:33:32 +0900 Subject: [PATCH 71/93] return sql.NullTime if it available (#1145) * return sql.NullTime if it available * NullTime should be used with parseTime=true option --- driver_test.go | 222 ++++++++++++++++++++++----------------------- fields.go | 2 +- nulltime_go113.go | 5 + nulltime_legacy.go | 5 + 4 files changed, 119 insertions(+), 115 deletions(-) diff --git a/driver_test.go b/driver_test.go index 34b476ed3..aa55d2f55 100644 --- a/driver_test.go +++ b/driver_test.go @@ -2758,13 +2758,13 @@ func TestRowsColumnTypes(t *testing.T) { nfNULL := sql.NullFloat64{Float64: 0.0, Valid: false} nf0 := sql.NullFloat64{Float64: 0.0, Valid: true} nf1337 := sql.NullFloat64{Float64: 13.37, Valid: true} - nt0 := NullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 0, time.UTC), Valid: true} - nt1 := NullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 100000000, time.UTC), Valid: true} - nt2 := NullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 110000000, time.UTC), Valid: true} - nt6 := NullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 111111000, time.UTC), Valid: true} - nd1 := NullTime{Time: time.Date(2006, 01, 02, 0, 0, 0, 0, time.UTC), Valid: true} - nd2 := NullTime{Time: time.Date(2006, 03, 04, 0, 0, 0, 0, time.UTC), Valid: true} - ndNULL := NullTime{Time: time.Time{}, Valid: false} + nt0 := nullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 0, time.UTC), Valid: true} + nt1 := nullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 100000000, time.UTC), Valid: true} + nt2 := nullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 110000000, time.UTC), Valid: true} + nt6 := nullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 111111000, time.UTC), Valid: true} + nd1 := nullTime{Time: time.Date(2006, 01, 02, 0, 0, 0, 0, time.UTC), Valid: true} + nd2 := nullTime{Time: time.Date(2006, 03, 04, 0, 0, 0, 0, time.UTC), Valid: true} + ndNULL := nullTime{Time: time.Time{}, Valid: false} rbNULL := sql.RawBytes(nil) rb0 := sql.RawBytes("0") rb42 := sql.RawBytes("42") @@ -2844,131 +2844,125 @@ func TestRowsColumnTypes(t *testing.T) { values2 = values2[:len(values2)-2] values3 = values3[:len(values3)-2] - dsns := []string{ - dsn + "&parseTime=true", - dsn + "&parseTime=false", - } - for _, testdsn := range dsns { - runTests(t, testdsn, func(dbt *DBTest) { - dbt.mustExec("CREATE TABLE test (" + schema + ")") - dbt.mustExec("INSERT INTO test VALUES (" + values1 + "), (" + values2 + "), (" + values3 + ")") + runTests(t, dsn+"&parseTime=true", func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (" + schema + ")") + dbt.mustExec("INSERT INTO test VALUES (" + values1 + "), (" + values2 + "), (" + values3 + ")") - rows, err := dbt.db.Query("SELECT * FROM test") - if err != nil { - t.Fatalf("Query: %v", err) - } + rows, err := dbt.db.Query("SELECT * FROM test") + if err != nil { + t.Fatalf("Query: %v", err) + } - tt, err := rows.ColumnTypes() - if err != nil { - t.Fatalf("ColumnTypes: %v", err) - } + tt, err := rows.ColumnTypes() + if err != nil { + t.Fatalf("ColumnTypes: %v", err) + } - if len(tt) != len(columns) { - t.Fatalf("unexpected number of columns: expected %d, got %d", len(columns), len(tt)) - } + if len(tt) != len(columns) { + t.Fatalf("unexpected number of columns: expected %d, got %d", len(columns), len(tt)) + } - types := make([]reflect.Type, len(tt)) - for i, tp := range tt { - column := columns[i] + types := make([]reflect.Type, len(tt)) + for i, tp := range tt { + column := columns[i] - // Name - name := tp.Name() - if name != column.name { - t.Errorf("column name mismatch %s != %s", name, column.name) - continue - } + // Name + name := tp.Name() + if name != column.name { + t.Errorf("column name mismatch %s != %s", name, column.name) + continue + } - // DatabaseTypeName - databaseTypeName := tp.DatabaseTypeName() - if databaseTypeName != column.databaseTypeName { - t.Errorf("databasetypename name mismatch for column %q: %s != %s", name, databaseTypeName, column.databaseTypeName) - continue - } + // DatabaseTypeName + databaseTypeName := tp.DatabaseTypeName() + if databaseTypeName != column.databaseTypeName { + t.Errorf("databasetypename name mismatch for column %q: %s != %s", name, databaseTypeName, column.databaseTypeName) + continue + } - // ScanType - scanType := tp.ScanType() - if scanType != column.scanType { - if scanType == nil { - t.Errorf("scantype is null for column %q", name) - } else { - t.Errorf("scantype mismatch for column %q: %s != %s", name, scanType.Name(), column.scanType.Name()) - } - continue + // ScanType + scanType := tp.ScanType() + if scanType != column.scanType { + if scanType == nil { + t.Errorf("scantype is null for column %q", name) + } else { + t.Errorf("scantype mismatch for column %q: %s != %s", name, scanType.Name(), column.scanType.Name()) } - types[i] = scanType - - // Nullable - nullable, ok := tp.Nullable() + continue + } + types[i] = scanType + + // Nullable + nullable, ok := tp.Nullable() + if !ok { + t.Errorf("nullable not ok %q", name) + continue + } + if nullable != column.nullable { + t.Errorf("nullable mismatch for column %q: %t != %t", name, nullable, column.nullable) + } + + // Length + // length, ok := tp.Length() + // if length != column.length { + // if !ok { + // t.Errorf("length not ok for column %q", name) + // } else { + // t.Errorf("length mismatch for column %q: %d != %d", name, length, column.length) + // } + // continue + // } + + // Precision and Scale + precision, scale, ok := tp.DecimalSize() + if precision != column.precision { if !ok { - t.Errorf("nullable not ok %q", name) - continue - } - if nullable != column.nullable { - t.Errorf("nullable mismatch for column %q: %t != %t", name, nullable, column.nullable) + t.Errorf("precision not ok for column %q", name) + } else { + t.Errorf("precision mismatch for column %q: %d != %d", name, precision, column.precision) } - - // Length - // length, ok := tp.Length() - // if length != column.length { - // if !ok { - // t.Errorf("length not ok for column %q", name) - // } else { - // t.Errorf("length mismatch for column %q: %d != %d", name, length, column.length) - // } - // continue - // } - - // Precision and Scale - precision, scale, ok := tp.DecimalSize() - if precision != column.precision { - if !ok { - t.Errorf("precision not ok for column %q", name) - } else { - t.Errorf("precision mismatch for column %q: %d != %d", name, precision, column.precision) - } - continue - } - if scale != column.scale { - if !ok { - t.Errorf("scale not ok for column %q", name) - } else { - t.Errorf("scale mismatch for column %q: %d != %d", name, scale, column.scale) - } - continue + continue + } + if scale != column.scale { + if !ok { + t.Errorf("scale not ok for column %q", name) + } else { + t.Errorf("scale mismatch for column %q: %d != %d", name, scale, column.scale) } + continue } + } - values := make([]interface{}, len(tt)) - for i := range values { - values[i] = reflect.New(types[i]).Interface() + values := make([]interface{}, len(tt)) + for i := range values { + values[i] = reflect.New(types[i]).Interface() + } + i := 0 + for rows.Next() { + err = rows.Scan(values...) + if err != nil { + t.Fatalf("failed to scan values in %v", err) } - i := 0 - for rows.Next() { - err = rows.Scan(values...) - if err != nil { - t.Fatalf("failed to scan values in %v", err) - } - for j := range values { - value := reflect.ValueOf(values[j]).Elem().Interface() - if !reflect.DeepEqual(value, columns[j].valuesOut[i]) { - if columns[j].scanType == scanTypeRawBytes { - t.Errorf("row %d, column %d: %v != %v", i, j, string(value.(sql.RawBytes)), string(columns[j].valuesOut[i].(sql.RawBytes))) - } else { - t.Errorf("row %d, column %d: %v != %v", i, j, value, columns[j].valuesOut[i]) - } + for j := range values { + value := reflect.ValueOf(values[j]).Elem().Interface() + if !reflect.DeepEqual(value, columns[j].valuesOut[i]) { + if columns[j].scanType == scanTypeRawBytes { + t.Errorf("row %d, column %d: %v != %v", i, j, string(value.(sql.RawBytes)), string(columns[j].valuesOut[i].(sql.RawBytes))) + } else { + t.Errorf("row %d, column %d: %v != %v", i, j, value, columns[j].valuesOut[i]) } } - i++ - } - if i != 3 { - t.Errorf("expected 3 rows, got %d", i) } + i++ + } + if i != 3 { + t.Errorf("expected 3 rows, got %d", i) + } - if err := rows.Close(); err != nil { - t.Errorf("error closing rows: %s", err) - } - }) - } + if err := rows.Close(); err != nil { + t.Errorf("error closing rows: %s", err) + } + }) } func TestValuerWithValueReceiverGivenNilValue(t *testing.T) { diff --git a/fields.go b/fields.go index e1e2ece4b..ed6c7a37d 100644 --- a/fields.go +++ b/fields.go @@ -106,7 +106,7 @@ var ( scanTypeInt64 = reflect.TypeOf(int64(0)) scanTypeNullFloat = reflect.TypeOf(sql.NullFloat64{}) scanTypeNullInt = reflect.TypeOf(sql.NullInt64{}) - scanTypeNullTime = reflect.TypeOf(NullTime{}) + scanTypeNullTime = reflect.TypeOf(nullTime{}) scanTypeUint8 = reflect.TypeOf(uint8(0)) scanTypeUint16 = reflect.TypeOf(uint16(0)) scanTypeUint32 = reflect.TypeOf(uint32(0)) diff --git a/nulltime_go113.go b/nulltime_go113.go index cb9bcae4a..453b4b394 100644 --- a/nulltime_go113.go +++ b/nulltime_go113.go @@ -33,3 +33,8 @@ import ( // NullTime.Scan interprets a time as UTC, not the loc DSN parameter. // Use sql.NullTime instead. type NullTime sql.NullTime + +// for internal use. +// the mysql package uses sql.NullTime if it is available. +// if not, the package uses mysql.NullTime. +type nullTime = sql.NullTime // sql.NullTime is available diff --git a/nulltime_legacy.go b/nulltime_legacy.go index 86d159d44..9f7ae27a8 100644 --- a/nulltime_legacy.go +++ b/nulltime_legacy.go @@ -32,3 +32,8 @@ type NullTime struct { Time time.Time Valid bool // Valid is true if Time is not NULL } + +// for internal use. +// the mysql package uses sql.NullTime if it is available. +// if not, the package uses mysql.NullTime. +type nullTime = NullTime // sql.NullTime is not available From e2ee2f35c7a3d5a37ddcbaff7f6b679ffd9e457c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 16 Aug 2020 17:27:32 +0900 Subject: [PATCH 72/93] README: Recommend some DB settings (#1141) --- README.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 834cf10f5..969aec554 100644 --- a/README.md +++ b/README.md @@ -56,15 +56,31 @@ Make sure [Git is installed](https://git-scm.com/downloads) on your machine and _Go MySQL Driver_ is an implementation of Go's `database/sql/driver` interface. You only need to import the driver and can use the full [`database/sql`](https://golang.org/pkg/database/sql/) API then. Use `mysql` as `driverName` and a valid [DSN](#dsn-data-source-name) as `dataSourceName`: + ```go import "database/sql" import _ "github.com/go-sql-driver/mysql" db, err := sql.Open("mysql", "user:password@/dbname") +if err != nil { + panic(err) +} +// See "Important settings" section. +db.SetConnMaxLifetime(time.Minute * 3) +db.SetMaxOpenConns(10) +db.SetMaxIdleConns(10) ``` [Examples are available in our Wiki](https://github.com/go-sql-driver/mysql/wiki/Examples "Go-MySQL-Driver Examples"). +### Important settings + +`db.SetConnMaxLifetime()` is required to ensure connections are closed by the driver safely before connection is closed by MySQL server, OS, or other middlewares. Since some middlewares close idle connections by 5 minutes, we recommend timeout shorter than 5 minutes. This setting helps load balancing and changing system variables too. + +`db.SetMaxOpenConns()` is highly recommended to limit the number of connection used by the application. There is no recommended limit number because it depends on application and MySQL server. + +`db.SetMaxIdleConns()` is recommended to be set same to (or greater than) `db.SetMaxOpenConns()`. When it is smaller than `SetMaxOpenConns()`, connections can be opened and closed very frequently than you expect. Idle connections can be closed by the `db.SetConnMaxLifetime()`. If you want to close idle connections more rapidly, you can use `db.SetConnMaxIdleTime()` since Go 1.15. + ### DSN (Data Source Name) @@ -496,4 +512,3 @@ Please read the [MPL 2.0 FAQ](https://www.mozilla.org/en-US/MPL/2.0/FAQ/) if you You can read the full terms here: [LICENSE](https://raw.github.com/go-sql-driver/mysql/master/LICENSE). ![Go Gopher and MySQL Dolphin](https://raw.github.com/wiki/go-sql-driver/mysql/go-mysql-driver_m.jpg "Golang Gopher transporting the MySQL Dolphin in a wheelbarrow") - From 46351a8892976898935f653f5333782579a96fa5 Mon Sep 17 00:00:00 2001 From: Jinhua Tan <312841925@qq.com> Date: Tue, 18 Aug 2020 19:12:13 +0800 Subject: [PATCH 73/93] auth: do not send 0 byte for mysql_old_password when password is empty (#1133) --- AUTHORS | 1 + auth.go | 7 +++---- auth_test.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/AUTHORS b/AUTHORS index ccf2f466c..112e07524 100644 --- a/AUTHORS +++ b/AUTHORS @@ -86,6 +86,7 @@ Soroush Pour Stan Putrya Stanley Gunawan Steven Hartland +Tan Jinhua <312841925 at qq.com> Thomas Wodarek Tim Ruffles Tom Jenkinson diff --git a/auth.go b/auth.go index fec7040d4..1f9ceb059 100644 --- a/auth.go +++ b/auth.go @@ -136,10 +136,6 @@ func pwHash(password []byte) (result [2]uint32) { // Hash password using insecure pre 4.1 method func scrambleOldPassword(scramble []byte, password string) []byte { - if len(password) == 0 { - return nil - } - scramble = scramble[:8] hashPw := pwHash([]byte(password)) @@ -247,6 +243,9 @@ func (mc *mysqlConn) auth(authData []byte, plugin string) ([]byte, error) { if !mc.cfg.AllowOldPasswords { return nil, ErrOldPassword } + if len(mc.cfg.Passwd) == 0 { + return nil, nil + } // Note: there are edge cases where this should work but doesn't; // this is currently "wontfix": // https://github.com/go-sql-driver/mysql/issues/184 diff --git a/auth_test.go b/auth_test.go index 1920ef39f..3bce7fe22 100644 --- a/auth_test.go +++ b/auth_test.go @@ -1157,7 +1157,7 @@ func TestAuthSwitchOldPasswordEmpty(t *testing.T) { t.Errorf("got error: %v", err) } - expectedReply := []byte{1, 0, 0, 3, 0} + expectedReply := []byte{0, 0, 0, 3} if !bytes.Equal(conn.written, expectedReply) { t.Errorf("got unexpected data: %v", conn.written) } @@ -1184,7 +1184,7 @@ func TestOldAuthSwitchPasswordEmpty(t *testing.T) { t.Errorf("got error: %v", err) } - expectedReply := []byte{1, 0, 0, 3, 0} + expectedReply := []byte{0, 0, 0, 3} if !bytes.Equal(conn.written, expectedReply) { t.Errorf("got unexpected data: %v", conn.written) } From cf72fd249522a0cc5e1ce727dfa15189dcb6411f Mon Sep 17 00:00:00 2001 From: Sho IIZUKA Date: Sat, 14 Nov 2020 10:11:01 +0900 Subject: [PATCH 74/93] Fix a broken link to cleartext client side plugin (#1165) --- AUTHORS | 1 + README.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 112e07524..3460c2e06 100644 --- a/AUTHORS +++ b/AUTHORS @@ -79,6 +79,7 @@ Reed Allman Richard Wilkes Robert Russell Runrioter Wung +Sho Iizuka Sho Ikeda Shuode Li Simon J Mudd diff --git a/README.md b/README.md index 969aec554..8f4c6fb04 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,7 @@ Valid Values: true, false Default: false ``` -`allowCleartextPasswords=true` allows using the [cleartext client side plugin](http://dev.mysql.com/doc/en/cleartext-authentication-plugin.html) if required by an account, such as one defined with the [PAM authentication plugin](http://dev.mysql.com/doc/en/pam-authentication-plugin.html). Sending passwords in clear text may be a security problem in some configurations. To avoid problems if there is any possibility that the password would be intercepted, clients should connect to MySQL Server using a method that protects the password. Possibilities include [TLS / SSL](#tls), IPsec, or a private network. +`allowCleartextPasswords=true` allows using the [cleartext client side plugin](https://dev.mysql.com/doc/en/cleartext-pluggable-authentication.html) if required by an account, such as one defined with the [PAM authentication plugin](http://dev.mysql.com/doc/en/pam-authentication-plugin.html). Sending passwords in clear text may be a security problem in some configurations. To avoid problems if there is any possibility that the password would be intercepted, clients should connect to MySQL Server using a method that protects the password. Possibilities include [TLS / SSL](#tls), IPsec, or a private network. ##### `allowNativePasswords` From aae55f736d467f81dced6ab1d37d4128209f0962 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 26 Nov 2020 11:36:41 +0900 Subject: [PATCH 75/93] README: Make usage code more friendly (#1170) --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8f4c6fb04..0b13154fc 100644 --- a/README.md +++ b/README.md @@ -58,8 +58,14 @@ _Go MySQL Driver_ is an implementation of Go's `database/sql/driver` interface. Use `mysql` as `driverName` and a valid [DSN](#dsn-data-source-name) as `dataSourceName`: ```go -import "database/sql" -import _ "github.com/go-sql-driver/mysql" +import ( + "database/sql" + "time" + + _ "github.com/go-sql-driver/mysql" +) + +// ... db, err := sql.Open("mysql", "user:password@/dbname") if err != nil { From f6dcc3d870a826fb4212f2f124acd2e4658632c5 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 27 Nov 2020 22:54:36 +0900 Subject: [PATCH 76/93] Fix go vet error (#1173) --- benchmark_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/benchmark_test.go b/benchmark_test.go index 3e25a3bf2..1030ddc52 100644 --- a/benchmark_test.go +++ b/benchmark_test.go @@ -127,7 +127,8 @@ func BenchmarkExec(b *testing.B) { } if _, err := stmt.Exec(); err != nil { - b.Fatal(err.Error()) + b.Logf("stmt.Exec failed: %v", err) + b.Fail() } } }() From a341cd15c45e4cdae93421b02834fdadf61e5e18 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 29 Dec 2020 00:21:29 +0900 Subject: [PATCH 77/93] Move tests from Travis to Actions (#1183) * Run macOS CI on Github Actions * no cover * Fix workflow name * Actions: Add linux test * fix * fix * fix go 1.10 * Remove .travis.yml --- .github/workflows/linux.yaml | 50 +++++++++++++ .github/workflows/mac.yaml | 36 +++++++++ .travis.yml | 138 ----------------------------------- 3 files changed, 86 insertions(+), 138 deletions(-) create mode 100644 .github/workflows/linux.yaml create mode 100644 .github/workflows/mac.yaml delete mode 100644 .travis.yml diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml new file mode 100644 index 000000000..c00bd72c4 --- /dev/null +++ b/.github/workflows/linux.yaml @@ -0,0 +1,50 @@ +name: Test on Linux +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-20.04 + strategy: + matrix: + go: [1.15, 1.14, 1.13, 1.12, 1.11, "1.10"] + mysql: ["mysql:8.0"] + include: + - go: 1.15 + mysql: "mysql:5.7" + - go: 1.15 + mysql: "mysql:5.5" + - go: 1.15 + mysql: "mariadb:5.5" + - go: 1.15 + mysql: "mariadb:10.1" + + services: + mysql: + image: ${{ matrix.mysql }} + ports: + - 3306:3306 + env: + MYSQL_DATABASE: gotest + MYSQL_USER: gotest + MYSQL_PASSWORD: secret + MYSQL_ALLOW_EMPTY_PASSWORD: yes + # How can we add these options? + #options: --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 + + steps: + - uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} + - uses: actions/checkout@v2 + - name: Run test + env: + MYSQL_TEST_USER: gotest + MYSQL_TEST_PASS: secret + MYSQL_TEST_ADDR: 127.0.0.1:3307 + MYSQL_TEST_CONCURRENT: 1 + run: | + go test -v -covermode=count -coverprofile=coverage.out + go vet ./... + gofmt -d -s . diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml new file mode 100644 index 000000000..f80a90b6e --- /dev/null +++ b/.github/workflows/mac.yaml @@ -0,0 +1,36 @@ +name: Test on macOS +on: + push: + pull_request: + +jobs: + build: + runs-on: macos-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up MySQL + run: | + brew install mysql + echo -e "[server]\ninnodb_log_file_size=256MB\ninnodb_buffer_pool_size=512MB\nmax_allowed_packet=16MB\nlocal_infile=1" >> /usr/local/etc/my.cnf + mysql.server start + mysql -uroot -e 'CREATE USER gotest IDENTIFIED BY "secret"' + mysql -uroot -e 'GRANT ALL ON *.* TO gotest' + mysql -uroot -e 'create database gotest;' + + - name: Set up Go + run: | + go install golang.org/x/tools/cmd/cover + + - name: Run tests + env: + MYSQL_TEST_USER: gotest + MYSQL_TEST_PASS: secret + MYSQL_TEST_ADDR: 127.0.0.1:3306 + MYSQL_TEST_CONCURRENT: 1 + run: | + go test -v -covermode=count -coverprofile=coverage.out + go vet ./... + gofmt -d -s . diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 2926bdd11..000000000 --- a/.travis.yml +++ /dev/null @@ -1,138 +0,0 @@ -language: go -go: - # Keep the most recent production release at the top - - 1.15.x - # Go development version - - master - # Older production releases - - 1.14.x - - 1.13.x - - 1.12.x - - 1.11.x - - 1.10.x - -before_install: - - go get golang.org/x/tools/cmd/cover - - go get github.com/mattn/goveralls - -before_script: - - echo -e "[server]\ninnodb_log_file_size=256MB\ninnodb_buffer_pool_size=512MB\nmax_allowed_packet=16MB" | sudo tee -a /etc/mysql/my.cnf - - sudo service mysql restart - - .travis/wait_mysql.sh - - mysql -e 'create database gotest;' - -jobs: - allow_failures: - - go: master - - include: - - env: DB=MYSQL8 - dist: xenial - go: 1.15.x - services: - - docker - before_install: - - go get golang.org/x/tools/cmd/cover - - go get github.com/mattn/goveralls - - docker pull mysql:8.0 - - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ALLOW_EMPTY_PASSWORD=yes - mysql:8.0 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 - - cp .travis/docker.cnf ~/.my.cnf - - .travis/wait_mysql.sh - before_script: - - export cross_compile=false - - export MYSQL_TEST_USER=gotest - - export MYSQL_TEST_PASS=secret - - export MYSQL_TEST_ADDR=127.0.0.1:3307 - - export MYSQL_TEST_CONCURRENT=1 - - - env: DB=MYSQL57 - dist: xenial - go: 1.15.x - services: - - docker - before_install: - - go get golang.org/x/tools/cmd/cover - - go get github.com/mattn/goveralls - - docker pull mysql:5.7 - - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ALLOW_EMPTY_PASSWORD=yes - mysql:5.7 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 - - cp .travis/docker.cnf ~/.my.cnf - - .travis/wait_mysql.sh - before_script: - - export cross_compile=false - - export MYSQL_TEST_USER=gotest - - export MYSQL_TEST_PASS=secret - - export MYSQL_TEST_ADDR=127.0.0.1:3307 - - export MYSQL_TEST_CONCURRENT=1 - - - env: DB=MARIA55 - dist: xenial - go: 1.15.x - services: - - docker - before_install: - - go get golang.org/x/tools/cmd/cover - - go get github.com/mattn/goveralls - - docker pull mariadb:5.5 - - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ALLOW_EMPTY_PASSWORD=yes - mariadb:5.5 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 - - cp .travis/docker.cnf ~/.my.cnf - - .travis/wait_mysql.sh - before_script: - - export cross_compile=false - - export MYSQL_TEST_USER=gotest - - export MYSQL_TEST_PASS=secret - - export MYSQL_TEST_ADDR=127.0.0.1:3307 - - export MYSQL_TEST_CONCURRENT=1 - - - env: DB=MARIA10_1 - dist: xenial - go: 1.15.x - services: - - docker - before_install: - - go get golang.org/x/tools/cmd/cover - - go get github.com/mattn/goveralls - - docker pull mariadb:10.1 - - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ALLOW_EMPTY_PASSWORD=yes - mariadb:10.1 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 - - cp .travis/docker.cnf ~/.my.cnf - - .travis/wait_mysql.sh - before_script: - - export cross_compile=false - - export MYSQL_TEST_USER=gotest - - export MYSQL_TEST_PASS=secret - - export MYSQL_TEST_ADDR=127.0.0.1:3307 - - export MYSQL_TEST_CONCURRENT=1 - - - os: osx - osx_image: xcode11.4 - addons: - homebrew: - packages: - - mysql - update: true - go: 1.15.x - before_install: - - go get golang.org/x/tools/cmd/cover - - go get github.com/mattn/goveralls - before_script: - - export cross_compile=false - - echo -e "[server]\ninnodb_log_file_size=256MB\ninnodb_buffer_pool_size=512MB\nmax_allowed_packet=16MB\nlocal_infile=1" >> /usr/local/etc/my.cnf - - mysql.server start - - mysql -uroot -e 'CREATE USER gotest IDENTIFIED BY "secret"' - - mysql -uroot -e 'GRANT ALL ON *.* TO gotest' - - mysql -uroot -e 'create database gotest;' - - export MYSQL_TEST_USER=gotest - - export MYSQL_TEST_PASS=secret - - export MYSQL_TEST_ADDR=127.0.0.1:3306 - - export MYSQL_TEST_CONCURRENT=1 - -script: - - go test -v -covermode=count -coverprofile=coverage.out - - go vet ./... - - .travis/gofmt.sh - - if [ "$cross_compile" != "false" ]; then .travis/compile_check.sh; fi -after_script: - - $HOME/gopath/bin/goveralls -coverprofile=coverage.out -service=travis-ci From 4d5208afde31873cb7e2c02c78cb58dd2b3cb6e0 Mon Sep 17 00:00:00 2001 From: Ichinose Shogo Date: Mon, 1 Feb 2021 23:46:06 +0900 Subject: [PATCH 78/93] improve GitHub Actions workflows (#1190) * improve GitHub Actions workflows - add windows to build matrix - use newer versions of MySQL and MariaDB * remove branch restriction --- .github/workflows/linux.yaml | 50 ----------------- .github/workflows/mac.yaml | 36 ------------ .github/workflows/test.yml | 104 +++++++++++++++++++++++++++++++++++ .travis/compile_check.sh | 17 ------ .travis/docker.cnf | 5 -- .travis/gofmt.sh | 7 --- .travis/wait_mysql.sh | 14 ----- driver_test.go | 13 +++++ 8 files changed, 117 insertions(+), 129 deletions(-) delete mode 100644 .github/workflows/linux.yaml delete mode 100644 .github/workflows/mac.yaml create mode 100644 .github/workflows/test.yml delete mode 100755 .travis/compile_check.sh delete mode 100644 .travis/docker.cnf delete mode 100755 .travis/gofmt.sh delete mode 100755 .travis/wait_mysql.sh diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml deleted file mode 100644 index c00bd72c4..000000000 --- a/.github/workflows/linux.yaml +++ /dev/null @@ -1,50 +0,0 @@ -name: Test on Linux -on: - push: - pull_request: - -jobs: - test: - runs-on: ubuntu-20.04 - strategy: - matrix: - go: [1.15, 1.14, 1.13, 1.12, 1.11, "1.10"] - mysql: ["mysql:8.0"] - include: - - go: 1.15 - mysql: "mysql:5.7" - - go: 1.15 - mysql: "mysql:5.5" - - go: 1.15 - mysql: "mariadb:5.5" - - go: 1.15 - mysql: "mariadb:10.1" - - services: - mysql: - image: ${{ matrix.mysql }} - ports: - - 3306:3306 - env: - MYSQL_DATABASE: gotest - MYSQL_USER: gotest - MYSQL_PASSWORD: secret - MYSQL_ALLOW_EMPTY_PASSWORD: yes - # How can we add these options? - #options: --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 - - steps: - - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go }} - - uses: actions/checkout@v2 - - name: Run test - env: - MYSQL_TEST_USER: gotest - MYSQL_TEST_PASS: secret - MYSQL_TEST_ADDR: 127.0.0.1:3307 - MYSQL_TEST_CONCURRENT: 1 - run: | - go test -v -covermode=count -coverprofile=coverage.out - go vet ./... - gofmt -d -s . diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml deleted file mode 100644 index f80a90b6e..000000000 --- a/.github/workflows/mac.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: Test on macOS -on: - push: - pull_request: - -jobs: - build: - runs-on: macos-latest - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up MySQL - run: | - brew install mysql - echo -e "[server]\ninnodb_log_file_size=256MB\ninnodb_buffer_pool_size=512MB\nmax_allowed_packet=16MB\nlocal_infile=1" >> /usr/local/etc/my.cnf - mysql.server start - mysql -uroot -e 'CREATE USER gotest IDENTIFIED BY "secret"' - mysql -uroot -e 'GRANT ALL ON *.* TO gotest' - mysql -uroot -e 'create database gotest;' - - - name: Set up Go - run: | - go install golang.org/x/tools/cmd/cover - - - name: Run tests - env: - MYSQL_TEST_USER: gotest - MYSQL_TEST_PASS: secret - MYSQL_TEST_ADDR: 127.0.0.1:3306 - MYSQL_TEST_CONCURRENT: 1 - run: | - go test -v -covermode=count -coverprofile=coverage.out - go vet ./... - gofmt -d -s . diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..99cf896a9 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,104 @@ +name: test +on: + pull_request: + push: + workflow_dispatch: + +env: + MYSQL_TEST_USER: gotest + MYSQL_TEST_PASS: secret + MYSQL_TEST_ADDR: 127.0.0.1:3306 + MYSQL_TEST_CONCURRENT: 1 + +jobs: + list: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: list + id: set-matrix + run: | + from __future__ import print_function + import json + go = [ + # Keep the most recent production release at the top + '1.15', + # Older production releases + '1.14', + '1.13', + '1.12', + '1.11', + ] + mysql = [ + '8.0', + '5.7', + '5.6', + 'mariadb-10.5', + 'mariadb-10.4', + 'mariadb-10.3', + ] + + includes = [] + # Go versions compatibility check + for v in go[1:]: + includes.append({'os': 'ubuntu-latest', 'go': v, 'mysql': mysql[0]}) + + matrix = { + # OS vs MySQL versions + 'os': [ 'ubuntu-latest', 'macos-latest', 'windows-latest' ], + 'go': [ go[0] ], + 'mysql': mysql, + + 'include': includes + } + output = json.dumps(matrix, separators=(',', ':')) + print('::set-output name=matrix::{0}'.format(output)) + shell: python + test: + needs: list + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.list.outputs.matrix) }} + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} + - uses: shogo82148/actions-setup-mysql@v1 + with: + mysql-version: ${{ matrix.mysql }} + user: ${{ env.MYSQL_TEST_USER }} + password: ${{ env.MYSQL_TEST_PASS }} + my-cnf: | + innodb_log_file_size=256MB + innodb_buffer_pool_size=512MB + max_allowed_packet=16MB + ; TestConcurrent fails if max_connections is too large + max_connections=50 + local_infile=1 + - name: setup database + run: | + mysql --user 'root' --host '127.0.0.1' -e 'create database gotest;' + + - name: test + run: | + go test -v '-covermode=count' '-coverprofile=coverage.out' + + - name: Send coverage + uses: shogo82148/actions-goveralls@v1 + with: + path-to-profile: coverage.out + flag-name: ${{ runner.os }}-Go-${{ matrix.go }}-DB-${{ matrix.mysql }} + parallel: true + + # notifies that all test jobs are finished. + finish: + needs: test + if: always() + runs-on: ubuntu-latest + steps: + - uses: shogo82148/actions-goveralls@v1 + with: + parallel-finished: true diff --git a/.travis/compile_check.sh b/.travis/compile_check.sh deleted file mode 100755 index 3bb3ed49d..000000000 --- a/.travis/compile_check.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -set -e -dist_list=$(go tool dist list) - -for dist in ${dist_list}; do - GOOS=$(echo ${dist} | cut -d "/" -f 1) - GOARCH=$(echo ${dist} | cut -d "/" -f 2) - set +e - GOOS=${GOOS} GOARCH=${GOARCH} go tool compile -V > /dev/null 2>&1 - if [[ $? -ne 0 ]]; then - echo "Compile support for ${GOOS}/${GOARCH} is not provided; skipping" - continue - fi - set -e - echo "Building ${GOOS}/${GOARCH}" - GOOS=${GOOS} GOARCH=${GOARCH} go build -o /dev/null - done diff --git a/.travis/docker.cnf b/.travis/docker.cnf deleted file mode 100644 index e57754e5a..000000000 --- a/.travis/docker.cnf +++ /dev/null @@ -1,5 +0,0 @@ -[client] -user = gotest -password = secret -host = 127.0.0.1 -port = 3307 diff --git a/.travis/gofmt.sh b/.travis/gofmt.sh deleted file mode 100755 index 9bf0d1684..000000000 --- a/.travis/gofmt.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -ev - -# Only check for go1.10+ since the gofmt style changed -if [[ $(go version) =~ go1\.([0-9]+) ]] && ((${BASH_REMATCH[1]} >= 10)); then - test -z "$(gofmt -d -s . | tee /dev/stderr)" -fi diff --git a/.travis/wait_mysql.sh b/.travis/wait_mysql.sh deleted file mode 100755 index fa2054ff1..000000000 --- a/.travis/wait_mysql.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -# use the mysql client inside the docker container if docker is running -[ "$(docker inspect -f '{{.State.Running}}' mysqld 2>/dev/null)" = "true" ] && mysql() { - docker exec mysqld mysql "${@}" -} - -while : -do - if mysql --protocol=tcp -e 'select version()'; then - break - fi - sleep 3 -done diff --git a/driver_test.go b/driver_test.go index aa55d2f55..54f7cd10c 100644 --- a/driver_test.go +++ b/driver_test.go @@ -24,6 +24,7 @@ import ( "net/url" "os" "reflect" + "runtime" "strings" "sync" "sync/atomic" @@ -1806,6 +1807,14 @@ func TestConcurrent(t *testing.T) { } runTests(t, dsn, func(dbt *DBTest) { + var version string + if err := dbt.db.QueryRow("SELECT @@version").Scan(&version); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if strings.Contains(strings.ToLower(version), "mariadb") { + t.Skip(`TODO: "fix commands out of sync. Did you run multiple statements at once?" on MariaDB`) + } + var max int err := dbt.db.QueryRow("SELECT @@max_connections").Scan(&max) if err != nil { @@ -2605,6 +2614,10 @@ func TestContextCancelStmtQuery(t *testing.T) { } func TestContextCancelBegin(t *testing.T) { + if runtime.GOOS == "windows" || runtime.GOOS == "darwin" { + t.Skip(`FIXME: it sometime fails with "expected driver.ErrBadConn, got sql: connection is already closed" on windows and macOS`) + } + runTests(t, dsn, func(dbt *DBTest) { dbt.mustExec("CREATE TABLE test (v INTEGER)") ctx, cancel := context.WithCancel(context.Background()) From fe2230a8b20cee1e48f7b75a9b363def5f950ba0 Mon Sep 17 00:00:00 2001 From: Ichinose Shogo Date: Tue, 2 Feb 2021 13:30:19 +0900 Subject: [PATCH 79/93] stop rounding times (#1172) * stop rounding times fixes https://github.com/go-sql-driver/mysql/issues/1121 * trim trailing zeros --- utils.go | 43 ++++++++++++++++++++++++++----------------- utils_test.go | 37 ++++++++++++++++++------------------- 2 files changed, 44 insertions(+), 36 deletions(-) diff --git a/utils.go b/utils.go index b0c6e9ca3..d6545f5be 100644 --- a/utils.go +++ b/utils.go @@ -277,15 +277,9 @@ func parseBinaryDateTime(num uint64, data []byte, loc *time.Location) (driver.Va } func appendDateTime(buf []byte, t time.Time) ([]byte, error) { - nsec := t.Nanosecond() - // to round under microsecond - if nsec%1000 >= 500 { // save half of time.Time.Add calls - t = t.Add(500 * time.Nanosecond) - nsec = t.Nanosecond() - } year, month, day := t.Date() hour, min, sec := t.Clock() - micro := nsec / 1000 + nsec := t.Nanosecond() if year < 1 || year > 9999 { return buf, errors.New("year is not in the range [1, 9999]: " + strconv.Itoa(year)) // use errors.New instead of fmt.Errorf to avoid year escape to heap @@ -293,14 +287,14 @@ func appendDateTime(buf []byte, t time.Time) ([]byte, error) { year100 := year / 100 year1 := year % 100 - var localBuf [26]byte // does not escape + var localBuf [len("2006-01-02T15:04:05.999999999")]byte // does not escape localBuf[0], localBuf[1], localBuf[2], localBuf[3] = digits10[year100], digits01[year100], digits10[year1], digits01[year1] localBuf[4] = '-' localBuf[5], localBuf[6] = digits10[month], digits01[month] localBuf[7] = '-' localBuf[8], localBuf[9] = digits10[day], digits01[day] - if hour == 0 && min == 0 && sec == 0 && micro == 0 { + if hour == 0 && min == 0 && sec == 0 && nsec == 0 { return append(buf, localBuf[:10]...), nil } @@ -311,18 +305,33 @@ func appendDateTime(buf []byte, t time.Time) ([]byte, error) { localBuf[16] = ':' localBuf[17], localBuf[18] = digits10[sec], digits01[sec] - if micro == 0 { + if nsec == 0 { return append(buf, localBuf[:19]...), nil } - - micro10000 := micro / 10000 - micro100 := (micro / 100) % 100 - micro1 := micro % 100 + nsec100000000 := nsec / 100000000 + nsec1000000 := (nsec / 1000000) % 100 + nsec10000 := (nsec / 10000) % 100 + nsec100 := (nsec / 100) % 100 + nsec1 := nsec % 100 localBuf[19] = '.' - localBuf[20], localBuf[21], localBuf[22], localBuf[23], localBuf[24], localBuf[25] = - digits10[micro10000], digits01[micro10000], digits10[micro100], digits01[micro100], digits10[micro1], digits01[micro1] - return append(buf, localBuf[:]...), nil + // milli second + localBuf[20], localBuf[21], localBuf[22] = + digits01[nsec100000000], digits10[nsec1000000], digits01[nsec1000000] + // micro second + localBuf[23], localBuf[24], localBuf[25] = + digits10[nsec10000], digits01[nsec10000], digits10[nsec100] + // nano second + localBuf[26], localBuf[27], localBuf[28] = + digits01[nsec100], digits10[nsec1], digits01[nsec1] + + // trim trailing zeros + n := len(localBuf) + for n > 0 && localBuf[n-1] == '0' { + n-- + } + + return append(buf, localBuf[:n]...), nil } // zeroDateTime is used in formatBinaryDateTime to avoid an allocation diff --git a/utils_test.go b/utils_test.go index e3619e7a7..67b132d2b 100644 --- a/utils_test.go +++ b/utils_test.go @@ -299,40 +299,40 @@ func TestAppendDateTime(t *testing.T) { str string }{ { - t: time.Date(2020, 05, 30, 0, 0, 0, 0, time.UTC), - str: "2020-05-30", + t: time.Date(1234, 5, 6, 0, 0, 0, 0, time.UTC), + str: "1234-05-06", }, { - t: time.Date(2020, 05, 30, 22, 0, 0, 0, time.UTC), - str: "2020-05-30 22:00:00", + t: time.Date(4567, 12, 31, 12, 0, 0, 0, time.UTC), + str: "4567-12-31 12:00:00", }, { - t: time.Date(2020, 05, 30, 22, 33, 0, 0, time.UTC), - str: "2020-05-30 22:33:00", + t: time.Date(2020, 5, 30, 12, 34, 0, 0, time.UTC), + str: "2020-05-30 12:34:00", }, { - t: time.Date(2020, 05, 30, 22, 33, 44, 0, time.UTC), - str: "2020-05-30 22:33:44", + t: time.Date(2020, 5, 30, 12, 34, 56, 0, time.UTC), + str: "2020-05-30 12:34:56", }, { - t: time.Date(2020, 05, 30, 22, 33, 44, 550000000, time.UTC), - str: "2020-05-30 22:33:44.550000", + t: time.Date(2020, 5, 30, 22, 33, 44, 123000000, time.UTC), + str: "2020-05-30 22:33:44.123", }, { - t: time.Date(2020, 05, 30, 22, 33, 44, 550000499, time.UTC), - str: "2020-05-30 22:33:44.550000", + t: time.Date(2020, 5, 30, 22, 33, 44, 123456000, time.UTC), + str: "2020-05-30 22:33:44.123456", }, { - t: time.Date(2020, 05, 30, 22, 33, 44, 550000500, time.UTC), - str: "2020-05-30 22:33:44.550001", + t: time.Date(2020, 5, 30, 22, 33, 44, 123456789, time.UTC), + str: "2020-05-30 22:33:44.123456789", }, { - t: time.Date(2020, 05, 30, 22, 33, 44, 550000567, time.UTC), - str: "2020-05-30 22:33:44.550001", + t: time.Date(9999, 12, 31, 23, 59, 59, 999999999, time.UTC), + str: "9999-12-31 23:59:59.999999999", }, { - t: time.Date(2020, 05, 30, 22, 33, 44, 999999567, time.UTC), - str: "2020-05-30 22:33:45", + t: time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC), + str: "0001-01-01", }, } for _, v := range tests { @@ -340,7 +340,6 @@ func TestAppendDateTime(t *testing.T) { buf, _ = appendDateTime(buf, v.t) if str := string(buf); str != v.str { t.Errorf("appendDateTime(%v), have: %s, want: %s", v.t, str, v.str) - return } } From 7b629a987258dcaba84d662edb5f408d11878fef Mon Sep 17 00:00:00 2001 From: MrBTTF Date: Mon, 1 Mar 2021 14:19:29 +0100 Subject: [PATCH 80/93] handling decoding pem error (#1192) --- AUTHORS | 1 + auth.go | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 3460c2e06..50afa2c85 100644 --- a/AUTHORS +++ b/AUTHORS @@ -92,6 +92,7 @@ Thomas Wodarek Tim Ruffles Tom Jenkinson Vladimir Kovpak +Vladyslav Zhelezniak Xiangyu Hu Xiaobing Jiang Xiuming Chen diff --git a/auth.go b/auth.go index 1f9ceb059..b2f19e8f0 100644 --- a/auth.go +++ b/auth.go @@ -15,6 +15,7 @@ import ( "crypto/sha256" "crypto/x509" "encoding/pem" + "fmt" "sync" ) @@ -371,7 +372,10 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { return err } - block, _ := pem.Decode(data[1:]) + block, rest := pem.Decode(data[1:]) + if block == nil { + return fmt.Errorf("No Pem data found, data: %s", rest) + } pkix, err := x509.ParsePKIXPublicKey(block.Bytes) if err != nil { return err From 33267428f77eb99023f936a2b9b330416053d67a Mon Sep 17 00:00:00 2001 From: Bui Quoc Trong Date: Tue, 2 Mar 2021 17:56:02 +0700 Subject: [PATCH 81/93] Implement driver.Validator interface (#1174) --- connection.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/connection.go b/connection.go index 90aec6439..835f89729 100644 --- a/connection.go +++ b/connection.go @@ -642,3 +642,9 @@ func (mc *mysqlConn) ResetSession(ctx context.Context) error { mc.reset = true return nil } + +// IsValid implements driver.Validator interface +// (From Go 1.15) +func (mc *mysqlConn) IsValid() bool { + return !mc.closed.IsSet() +} From e246959aa61ad88a4d5d459aae5e73d04faf8f85 Mon Sep 17 00:00:00 2001 From: Ichinose Shogo Date: Tue, 30 Mar 2021 08:00:02 +0900 Subject: [PATCH 82/93] add Go 1.16 to the build matrix (#1198) --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 99cf896a9..886002143 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,12 +19,12 @@ jobs: - name: list id: set-matrix run: | - from __future__ import print_function import json go = [ # Keep the most recent production release at the top - '1.15', + '1.16', # Older production releases + '1.15', '1.14', '1.13', '1.12', From bcc459a906419e2890a50fc2c99ea6dd927a88f2 Mon Sep 17 00:00:00 2001 From: Ichinose Shogo Date: Thu, 1 Apr 2021 10:56:49 +0900 Subject: [PATCH 83/93] Release v1.6.0 (#1197) * Release v1.6.0 * v1.6.0 is released on 2021-04-01 --- CHANGELOG.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cb97b38d..72a738ed5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,29 @@ +## Version 1.6 (2021-04-01) + +Changes: + + - Migrate the CI service from travis-ci to GitHub Actions (#1176, #1183, #1190) + - `NullTime` is deprecated (#960, #1144) + - Reduce allocations when building SET command (#1111) + - Performance improvement for time formatting (#1118) + - Performance improvement for time parsing (#1098, #1113) + +New Features: + + - Implement `driver.Validator` interface (#1106, #1174) + - Support returning `uint64` from `Valuer` in `ConvertValue` (#1143) + - Add `json.RawMessage` for converter and prepared statement (#1059) + - Interpolate `json.RawMessage` as `string` (#1058) + - Implements `CheckNamedValue` (#1090) + +Bugfixes: + + - Stop rounding times (#1121, #1172) + - Put zero filler into the SSL handshake packet (#1066) + - Fix checking cancelled connections back into the connection pool (#1095) + - Fix remove last 0 byte for mysql_old_password when password is empty (#1133) + + ## Version 1.5 (2020-01-07) Changes: From b36cd86ebcb680d317f962679780f5877a0b91e1 Mon Sep 17 00:00:00 2001 From: Ichinose Shogo Date: Tue, 18 May 2021 10:33:08 +0900 Subject: [PATCH 84/93] Drop support of Go 1.12 (#1211) * Drop support of Go 1.12 * bump Go version in go.mod * remove nulltime_legacy --- .github/workflows/test.yml | 2 -- README.md | 2 +- driver_test.go | 14 ++++++------- fields.go | 2 +- go.mod | 2 +- nulltime.go | 21 ++++++++++++++++++++ nulltime_go113.go | 40 -------------------------------------- nulltime_legacy.go | 39 ------------------------------------- 8 files changed, 31 insertions(+), 91 deletions(-) delete mode 100644 nulltime_go113.go delete mode 100644 nulltime_legacy.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 886002143..fce4cf670 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,8 +27,6 @@ jobs: '1.15', '1.14', '1.13', - '1.12', - '1.11', ] mysql = [ '8.0', diff --git a/README.md b/README.md index 0b13154fc..9c8284cd1 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) pac * Optional placeholder interpolation ## Requirements - * Go 1.10 or higher. We aim to support the 3 latest versions of Go. + * Go 1.13 or higher. We aim to support the 3 latest versions of Go. * MySQL (4.1+), MariaDB, Percona Server, Google CloudSQL or Sphinx (2.2.3+) --------------------------------------- diff --git a/driver_test.go b/driver_test.go index 54f7cd10c..3ae379b26 100644 --- a/driver_test.go +++ b/driver_test.go @@ -2771,13 +2771,13 @@ func TestRowsColumnTypes(t *testing.T) { nfNULL := sql.NullFloat64{Float64: 0.0, Valid: false} nf0 := sql.NullFloat64{Float64: 0.0, Valid: true} nf1337 := sql.NullFloat64{Float64: 13.37, Valid: true} - nt0 := nullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 0, time.UTC), Valid: true} - nt1 := nullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 100000000, time.UTC), Valid: true} - nt2 := nullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 110000000, time.UTC), Valid: true} - nt6 := nullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 111111000, time.UTC), Valid: true} - nd1 := nullTime{Time: time.Date(2006, 01, 02, 0, 0, 0, 0, time.UTC), Valid: true} - nd2 := nullTime{Time: time.Date(2006, 03, 04, 0, 0, 0, 0, time.UTC), Valid: true} - ndNULL := nullTime{Time: time.Time{}, Valid: false} + nt0 := sql.NullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 0, time.UTC), Valid: true} + nt1 := sql.NullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 100000000, time.UTC), Valid: true} + nt2 := sql.NullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 110000000, time.UTC), Valid: true} + nt6 := sql.NullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 111111000, time.UTC), Valid: true} + nd1 := sql.NullTime{Time: time.Date(2006, 01, 02, 0, 0, 0, 0, time.UTC), Valid: true} + nd2 := sql.NullTime{Time: time.Date(2006, 03, 04, 0, 0, 0, 0, time.UTC), Valid: true} + ndNULL := sql.NullTime{Time: time.Time{}, Valid: false} rbNULL := sql.RawBytes(nil) rb0 := sql.RawBytes("0") rb42 := sql.RawBytes("42") diff --git a/fields.go b/fields.go index ed6c7a37d..d82154ce8 100644 --- a/fields.go +++ b/fields.go @@ -106,7 +106,7 @@ var ( scanTypeInt64 = reflect.TypeOf(int64(0)) scanTypeNullFloat = reflect.TypeOf(sql.NullFloat64{}) scanTypeNullInt = reflect.TypeOf(sql.NullInt64{}) - scanTypeNullTime = reflect.TypeOf(nullTime{}) + scanTypeNullTime = reflect.TypeOf(sql.NullTime{}) scanTypeUint8 = reflect.TypeOf(uint8(0)) scanTypeUint16 = reflect.TypeOf(uint16(0)) scanTypeUint32 = reflect.TypeOf(uint32(0)) diff --git a/go.mod b/go.mod index fffbf6a90..251110478 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/go-sql-driver/mysql -go 1.10 +go 1.13 diff --git a/nulltime.go b/nulltime.go index 651723a96..17af92ddc 100644 --- a/nulltime.go +++ b/nulltime.go @@ -9,11 +9,32 @@ package mysql import ( + "database/sql" "database/sql/driver" "fmt" "time" ) +// NullTime represents a time.Time that may be NULL. +// NullTime implements the Scanner interface so +// it can be used as a scan destination: +// +// var nt NullTime +// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) +// ... +// if nt.Valid { +// // use nt.Time +// } else { +// // NULL value +// } +// +// This NullTime implementation is not driver-specific +// +// Deprecated: NullTime doesn't honor the loc DSN parameter. +// NullTime.Scan interprets a time as UTC, not the loc DSN parameter. +// Use sql.NullTime instead. +type NullTime sql.NullTime + // Scan implements the Scanner interface. // The value type must be time.Time or string / []byte (formatted time-string), // otherwise Scan fails. diff --git a/nulltime_go113.go b/nulltime_go113.go deleted file mode 100644 index 453b4b394..000000000 --- a/nulltime_go113.go +++ /dev/null @@ -1,40 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -// +build go1.13 - -package mysql - -import ( - "database/sql" -) - -// NullTime represents a time.Time that may be NULL. -// NullTime implements the Scanner interface so -// it can be used as a scan destination: -// -// var nt NullTime -// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) -// ... -// if nt.Valid { -// // use nt.Time -// } else { -// // NULL value -// } -// -// This NullTime implementation is not driver-specific -// -// Deprecated: NullTime doesn't honor the loc DSN parameter. -// NullTime.Scan interprets a time as UTC, not the loc DSN parameter. -// Use sql.NullTime instead. -type NullTime sql.NullTime - -// for internal use. -// the mysql package uses sql.NullTime if it is available. -// if not, the package uses mysql.NullTime. -type nullTime = sql.NullTime // sql.NullTime is available diff --git a/nulltime_legacy.go b/nulltime_legacy.go deleted file mode 100644 index 9f7ae27a8..000000000 --- a/nulltime_legacy.go +++ /dev/null @@ -1,39 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -// +build !go1.13 - -package mysql - -import ( - "time" -) - -// NullTime represents a time.Time that may be NULL. -// NullTime implements the Scanner interface so -// it can be used as a scan destination: -// -// var nt NullTime -// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) -// ... -// if nt.Valid { -// // use nt.Time -// } else { -// // NULL value -// } -// -// This NullTime implementation is not driver-specific -type NullTime struct { - Time time.Time - Valid bool // Valid is true if Time is not NULL -} - -// for internal use. -// the mysql package uses sql.NullTime if it is available. -// if not, the package uses mysql.NullTime. -type nullTime = NullTime // sql.NullTime is not available From 9942e21775f58b4a6339c3a814f7c1865d2007d9 Mon Sep 17 00:00:00 2001 From: lowang-bh Date: Wed, 26 May 2021 13:13:55 +0800 Subject: [PATCH 85/93] Fix readme: MaxIdle is same or less than MaxOpen (#1215) Co-authored-by: wanglonghui7 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9c8284cd1..eb9614cbd 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ db.SetMaxIdleConns(10) `db.SetMaxOpenConns()` is highly recommended to limit the number of connection used by the application. There is no recommended limit number because it depends on application and MySQL server. -`db.SetMaxIdleConns()` is recommended to be set same to (or greater than) `db.SetMaxOpenConns()`. When it is smaller than `SetMaxOpenConns()`, connections can be opened and closed very frequently than you expect. Idle connections can be closed by the `db.SetConnMaxLifetime()`. If you want to close idle connections more rapidly, you can use `db.SetConnMaxIdleTime()` since Go 1.15. +`db.SetMaxIdleConns()` is recommended to be set same to `db.SetMaxOpenConns()`. When it is smaller than `SetMaxOpenConns()`, connections can be opened and closed very frequently than you expect. Idle connections can be closed by the `db.SetConnMaxLifetime()`. If you want to close idle connections more rapidly, you can use `db.SetConnMaxIdleTime()` since Go 1.15. ### DSN (Data Source Name) From 4b653727060f5799e4e4bd96ad693046885b375e Mon Sep 17 00:00:00 2001 From: Ichinose Shogo Date: Wed, 26 May 2021 20:46:06 +0900 Subject: [PATCH 86/93] noCopy implements sync.Locker (#1216) noCopy is used by -copylocks checker from `go vet`. see https://github.com/golang/go/issues/8005#issuecomment-190753527 for details. but it doesn't work from Go 1.11, because of https://github.com/golang/go/commit/c2eba53e7f80df21d51285879d51ab81bcfbf6bc and https://github.com/golang/go/issues/26165 -copylock now works with structs that implement sync.Locker. So, noCopy should implement sync.Locker. --- utils.go | 6 ++++++ utils_test.go | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/utils.go b/utils.go index d6545f5be..b400cf99f 100644 --- a/utils.go +++ b/utils.go @@ -790,6 +790,12 @@ type noCopy struct{} // Lock is a no-op used by -copylocks checker from `go vet`. func (*noCopy) Lock() {} +// Unlock is a no-op used by -copylocks checker from `go vet`. +// noCopy should implement sync.Locker from Go 1.11 +// https://github.com/golang/go/commit/c2eba53e7f80df21d51285879d51ab81bcfbf6bc +// https://github.com/golang/go/issues/26165 +func (*noCopy) Unlock() {} + // atomicBool is a wrapper around uint32 for usage as a boolean value with // atomic access. type atomicBool struct { diff --git a/utils_test.go b/utils_test.go index 67b132d2b..b0069251e 100644 --- a/utils_test.go +++ b/utils_test.go @@ -228,7 +228,9 @@ func TestAtomicBool(t *testing.T) { t.Fatal("Expected value to be false") } - ab._noCopy.Lock() // we've "tested" it ¯\_(ツ)_/¯ + // we've "tested" them ¯\_(ツ)_/¯ + ab._noCopy.Lock() + defer ab._noCopy.Unlock() } func TestAtomicError(t *testing.T) { From e74ba5c13fd26850019c51afb1c4acf71f688aec Mon Sep 17 00:00:00 2001 From: Janek Vedock <83283832+jvedock@users.noreply.github.com> Date: Thu, 3 Jun 2021 04:45:34 -0400 Subject: [PATCH 87/93] Wording correction in README (#1218) * fixed grammatical error * Update AUTHORS --- AUTHORS | 1 + README.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 50afa2c85..900dfec06 100644 --- a/AUTHORS +++ b/AUTHORS @@ -45,6 +45,7 @@ Ilia Cimpoes INADA Naoki Jacek Szwec James Harr +Janek Vedock Jeff Hodges Jeffrey Charles Jerome Meyer diff --git a/README.md b/README.md index eb9614cbd..f056e614b 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ db.SetMaxIdleConns(10) `db.SetMaxOpenConns()` is highly recommended to limit the number of connection used by the application. There is no recommended limit number because it depends on application and MySQL server. -`db.SetMaxIdleConns()` is recommended to be set same to `db.SetMaxOpenConns()`. When it is smaller than `SetMaxOpenConns()`, connections can be opened and closed very frequently than you expect. Idle connections can be closed by the `db.SetConnMaxLifetime()`. If you want to close idle connections more rapidly, you can use `db.SetConnMaxIdleTime()` since Go 1.15. +`db.SetMaxIdleConns()` is recommended to be set same to `db.SetMaxOpenConns()`. When it is smaller than `SetMaxOpenConns()`, connections can be opened and closed much more frequently than you expect. Idle connections can be closed by the `db.SetConnMaxLifetime()`. If you want to close idle connections more rapidly, you can use `db.SetConnMaxIdleTime()` since Go 1.15. ### DSN (Data Source Name) From 417641ad42910e50863a0caf4740ce319262f2f9 Mon Sep 17 00:00:00 2001 From: Chris Kirkland Date: Thu, 3 Jun 2021 18:34:57 -0500 Subject: [PATCH 88/93] support Is comparison on MySQLError (#1210) * support Is comparison on MySQLError * add myself to authors * skip error tests for go 1.12 * remove test build tag --- AUTHORS | 1 + errors.go | 7 +++++++ errors_test.go | 19 +++++++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/AUTHORS b/AUTHORS index 900dfec06..e3370e025 100644 --- a/AUTHORS +++ b/AUTHORS @@ -23,6 +23,7 @@ Asta Xie Bulat Gaifullin Caine Jette Carlos Nieto +Chris Kirkland Chris Moos Craig Wilson Daniel Montoya diff --git a/errors.go b/errors.go index 760782ff2..92cc9a361 100644 --- a/errors.go +++ b/errors.go @@ -63,3 +63,10 @@ type MySQLError struct { func (me *MySQLError) Error() string { return fmt.Sprintf("Error %d: %s", me.Number, me.Message) } + +func (me *MySQLError) Is(err error) bool { + if merr, ok := err.(*MySQLError); ok { + return merr.Number == me.Number + } + return false +} diff --git a/errors_test.go b/errors_test.go index 96f9126d6..3a1aef74d 100644 --- a/errors_test.go +++ b/errors_test.go @@ -10,6 +10,7 @@ package mysql import ( "bytes" + "errors" "log" "testing" ) @@ -40,3 +41,21 @@ func TestErrorsStrictIgnoreNotes(t *testing.T) { dbt.mustExec("DROP TABLE IF EXISTS does_not_exist") }) } + +func TestMySQLErrIs(t *testing.T) { + infraErr := &MySQLError{1234, "the server is on fire"} + otherInfraErr := &MySQLError{1234, "the datacenter is flooded"} + if !errors.Is(infraErr, otherInfraErr) { + t.Errorf("expected errors to be the same: %+v %+v", infraErr, otherInfraErr) + } + + differentCodeErr := &MySQLError{5678, "the server is on fire"} + if errors.Is(infraErr, differentCodeErr) { + t.Fatalf("expected errors to be different: %+v %+v", infraErr, differentCodeErr) + } + + nonMysqlErr := errors.New("not a mysql error") + if errors.Is(infraErr, nonMysqlErr) { + t.Fatalf("expected errors to be different: %+v %+v", infraErr, nonMysqlErr) + } +} From 21f789cd2353b7ac81538f41426e9cfd2b1fcc87 Mon Sep 17 00:00:00 2001 From: ziheng Date: Wed, 30 Jun 2021 08:27:49 +0800 Subject: [PATCH 89/93] improve readability follows go-staticcheck (#1227) sign in AUTHORS --- AUTHORS | 1 + dsn.go | 1 - statement_test.go | 2 +- utils.go | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) diff --git a/AUTHORS b/AUTHORS index e3370e025..fee2d5ccf 100644 --- a/AUTHORS +++ b/AUTHORS @@ -101,6 +101,7 @@ Xiuming Chen Xuehong Chan Zhenye Xie Zhixin Wen +Ziheng Lyu # Organizations diff --git a/dsn.go b/dsn.go index 93f3548cb..a306d66a3 100644 --- a/dsn.go +++ b/dsn.go @@ -426,7 +426,6 @@ func parseDSNParams(cfg *Config, params string) (err error) { // Collation case "collation": cfg.Collation = value - break case "columnsWithAlias": var isBool bool diff --git a/statement_test.go b/statement_test.go index ac6b92de9..2563ece55 100644 --- a/statement_test.go +++ b/statement_test.go @@ -36,7 +36,7 @@ func TestConvertDerivedByteSlice(t *testing.T) { t.Fatal("Byte slice not convertible", err) } - if bytes.Compare(output.([]byte), []byte("value")) != 0 { + if !bytes.Equal(output.([]byte), []byte("value")) { t.Fatalf("Byte slice not converted, got %#v %T", output, output) } } diff --git a/utils.go b/utils.go index b400cf99f..bcdee1b46 100644 --- a/utils.go +++ b/utils.go @@ -199,7 +199,7 @@ func parseByteYear(b []byte) (int, error) { return 0, err } year += v * n - n = n / 10 + n /= 10 } return year, nil } From a34e090a4648ec0ec682e87966cdcd4e43006a79 Mon Sep 17 00:00:00 2001 From: Ichinose Shogo Date: Mon, 12 Jul 2021 13:36:52 +0900 Subject: [PATCH 90/93] use utf8mb4 instead of utf8 in TestCharset (#1228) From MySQL 8.0.24, `SELECT @@character_set_connection` reports utf8mb3 or utf8mb4 instead of utf8. Because utf8 is currently an alias for utf8mb3, however at some point utf8 is expected to become a reference to utf8mb4. > ref. https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-24.html#mysqld-8-0-24-bug > Important Note: When a utf8mb3 collation was specified in a CREATE TABLE statement, SHOW CREATE TABLE, DEFAULT CHARSET, > the values of system variables containing character set names, > and the binary log all subsequently displayed the character set as utf8 which is becoming a synonym for utf8mb4. > Now in such cases, utf8mb3 is shown instead, and CREATE TABLE raises the warning 'collation_name' is a collation of the deprecated character set UTF8MB3. > Please consider using UTF8MB4 with an appropriate collation instead. (Bug #27225287, Bug #32085357, Bug #32122844) > > References: See also: Bug #30624990. The document says that we should use utf8mb4 instead of utf8, so we should follow it. --- driver_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/driver_test.go b/driver_test.go index 3ae379b26..f1e4ad71e 100644 --- a/driver_test.go +++ b/driver_test.go @@ -1450,11 +1450,11 @@ func TestCharset(t *testing.T) { mustSetCharset("charset=ascii", "ascii") // when the first charset is invalid, use the second - mustSetCharset("charset=none,utf8", "utf8") + mustSetCharset("charset=none,utf8mb4", "utf8mb4") // when the first charset is valid, use it - mustSetCharset("charset=ascii,utf8", "ascii") - mustSetCharset("charset=utf8,ascii", "utf8") + mustSetCharset("charset=ascii,utf8mb4", "ascii") + mustSetCharset("charset=utf8mb4,ascii", "utf8mb4") } func TestFailingCharset(t *testing.T) { From 75d09acc46ea1a7074058d31da50293052248047 Mon Sep 17 00:00:00 2001 From: ziheng Date: Mon, 12 Jul 2021 17:58:03 +0800 Subject: [PATCH 91/93] refactoring (*textRows).readRow in a more clear way (#1230) --- packets.go | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/packets.go b/packets.go index 6664e5ae5..1867ecab2 100644 --- a/packets.go +++ b/packets.go @@ -761,40 +761,40 @@ func (rows *textRows) readRow(dest []driver.Value) error { } // RowSet Packet - var n int - var isNull bool - pos := 0 + var ( + n int + isNull bool + pos int = 0 + ) for i := range dest { // Read bytes and convert to string dest[i], isNull, n, err = readLengthEncodedString(data[pos:]) pos += n - if err == nil { - if !isNull { - if !mc.parseTime { - continue - } else { - switch rows.rs.columns[i].fieldType { - case fieldTypeTimestamp, fieldTypeDateTime, - fieldTypeDate, fieldTypeNewDate: - dest[i], err = parseDateTime( - dest[i].([]byte), - mc.cfg.Loc, - ) - if err == nil { - continue - } - default: - continue - } - } - } else { - dest[i] = nil - continue + if err != nil { + return err + } + + if isNull { + dest[i] = nil + continue + } + + if !mc.parseTime { + continue + } + + // Parse time field + switch rows.rs.columns[i].fieldType { + case fieldTypeTimestamp, + fieldTypeDateTime, + fieldTypeDate, + fieldTypeNewDate: + if dest[i], err = parseDateTime(dest[i].([]byte), mc.cfg.Loc); err != nil { + return err } } - return err // err != nil } return nil From 6a88ab97c64c79be016f34d11e3295b8d291d50b Mon Sep 17 00:00:00 2001 From: ziheng Date: Mon, 19 Jul 2021 07:00:59 +0800 Subject: [PATCH 92/93] add an invalid DSN test case (#1235) --- dsn_test.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/dsn_test.go b/dsn_test.go index 89815b341..fc6eea9c8 100644 --- a/dsn_test.go +++ b/dsn_test.go @@ -92,13 +92,14 @@ func TestDSNParser(t *testing.T) { func TestDSNParserInvalid(t *testing.T) { var invalidDSNs = []string{ - "@net(addr/", // no closing brace - "@tcp(/", // no closing brace - "tcp(/", // no closing brace - "(/", // no closing brace - "net(addr)//", // unescaped - "User:pass@tcp(1.2.3.4:3306)", // no trailing slash - "net()/", // unknown default addr + "@net(addr/", // no closing brace + "@tcp(/", // no closing brace + "tcp(/", // no closing brace + "(/", // no closing brace + "net(addr)//", // unescaped + "User:pass@tcp(1.2.3.4:3306)", // no trailing slash + "net()/", // unknown default addr + "user:pass@tcp(127.0.0.1:3306)/db/name", // invalid dbname //"/dbname?arg=/some/unescaped/path", } From e8f8fcdbd3e06fc74b6cf745c13eaebedff68c49 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 19 Jul 2021 07:01:45 +0800 Subject: [PATCH 93/93] return unsigned in database type name when necessary (#1238) * return unsigned in database type name when necessary * Fix test * Update README * Add myself in AUTHORS --- AUTHORS | 1 + README.md | 2 +- driver_test.go | 8 ++++---- fields.go | 12 ++++++++++++ 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/AUTHORS b/AUTHORS index fee2d5ccf..80199cf2c 100644 --- a/AUTHORS +++ b/AUTHORS @@ -67,6 +67,7 @@ Linh Tran Tuan Lion Yang Luca Looz Lucas Liu +Lunny Xiao Luke Scott Maciej Zimnoch Michael Woolnough diff --git a/README.md b/README.md index f056e614b..ded6e3b16 100644 --- a/README.md +++ b/README.md @@ -454,7 +454,7 @@ user:password@/ The connection pool is managed by Go's database/sql package. For details on how to configure the size of the pool and how long connections stay in the pool see `*DB.SetMaxOpenConns`, `*DB.SetMaxIdleConns`, and `*DB.SetConnMaxLifetime` in the [database/sql documentation](https://golang.org/pkg/database/sql/). The read, write, and dial timeouts for each individual connection are configured with the DSN parameters [`readTimeout`](#readtimeout), [`writeTimeout`](#writetimeout), and [`timeout`](#timeout), respectively. ## `ColumnType` Support -This driver supports the [`ColumnType` interface](https://golang.org/pkg/database/sql/#ColumnType) introduced in Go 1.8, with the exception of [`ColumnType.Length()`](https://golang.org/pkg/database/sql/#ColumnType.Length), which is currently not supported. +This driver supports the [`ColumnType` interface](https://golang.org/pkg/database/sql/#ColumnType) introduced in Go 1.8, with the exception of [`ColumnType.Length()`](https://golang.org/pkg/database/sql/#ColumnType.Length), which is currently not supported. All Unsigned database type names will be returned `UNSIGNED ` with `INT`, `TINYINT`, `SMALLINT`, `BIGINT`. ## `context.Context` Support Go 1.8 added `database/sql` support for `context.Context`. This driver supports query timeouts and cancellation via contexts. diff --git a/driver_test.go b/driver_test.go index f1e4ad71e..7b4ab84f3 100644 --- a/driver_test.go +++ b/driver_test.go @@ -2808,10 +2808,10 @@ func TestRowsColumnTypes(t *testing.T) { {"mediumintnull", "MEDIUMINT", "MEDIUMINT", scanTypeNullInt, true, 0, 0, [3]string{"0", "42", "NULL"}, [3]interface{}{ni0, ni42, niNULL}}, {"bigint", "BIGINT NOT NULL", "BIGINT", scanTypeInt64, false, 0, 0, [3]string{"0", "65535", "-42"}, [3]interface{}{int64(0), int64(65535), int64(-42)}}, {"bigintnull", "BIGINT", "BIGINT", scanTypeNullInt, true, 0, 0, [3]string{"NULL", "1", "42"}, [3]interface{}{niNULL, ni1, ni42}}, - {"tinyuint", "TINYINT UNSIGNED NOT NULL", "TINYINT", scanTypeUint8, false, 0, 0, [3]string{"0", "255", "42"}, [3]interface{}{uint8(0), uint8(255), uint8(42)}}, - {"smalluint", "SMALLINT UNSIGNED NOT NULL", "SMALLINT", scanTypeUint16, false, 0, 0, [3]string{"0", "65535", "42"}, [3]interface{}{uint16(0), uint16(65535), uint16(42)}}, - {"biguint", "BIGINT UNSIGNED NOT NULL", "BIGINT", scanTypeUint64, false, 0, 0, [3]string{"0", "65535", "42"}, [3]interface{}{uint64(0), uint64(65535), uint64(42)}}, - {"uint13", "INT(13) UNSIGNED NOT NULL", "INT", scanTypeUint32, false, 0, 0, [3]string{"0", "1337", "42"}, [3]interface{}{uint32(0), uint32(1337), uint32(42)}}, + {"tinyuint", "TINYINT UNSIGNED NOT NULL", "UNSIGNED TINYINT", scanTypeUint8, false, 0, 0, [3]string{"0", "255", "42"}, [3]interface{}{uint8(0), uint8(255), uint8(42)}}, + {"smalluint", "SMALLINT UNSIGNED NOT NULL", "UNSIGNED SMALLINT", scanTypeUint16, false, 0, 0, [3]string{"0", "65535", "42"}, [3]interface{}{uint16(0), uint16(65535), uint16(42)}}, + {"biguint", "BIGINT UNSIGNED NOT NULL", "UNSIGNED BIGINT", scanTypeUint64, false, 0, 0, [3]string{"0", "65535", "42"}, [3]interface{}{uint64(0), uint64(65535), uint64(42)}}, + {"uint13", "INT(13) UNSIGNED NOT NULL", "UNSIGNED INT", scanTypeUint32, false, 0, 0, [3]string{"0", "1337", "42"}, [3]interface{}{uint32(0), uint32(1337), uint32(42)}}, {"float", "FLOAT NOT NULL", "FLOAT", scanTypeFloat32, false, math.MaxInt64, math.MaxInt64, [3]string{"0", "42", "13.37"}, [3]interface{}{float32(0), float32(42), float32(13.37)}}, {"floatnull", "FLOAT", "FLOAT", scanTypeNullFloat, true, math.MaxInt64, math.MaxInt64, [3]string{"0", "NULL", "13.37"}, [3]interface{}{nf0, nfNULL, nf1337}}, {"float74null", "FLOAT(7,4)", "FLOAT", scanTypeNullFloat, true, math.MaxInt64, 4, [3]string{"0", "NULL", "13.37"}, [3]interface{}{nf0, nfNULL, nf1337}}, diff --git a/fields.go b/fields.go index d82154ce8..e0654a83d 100644 --- a/fields.go +++ b/fields.go @@ -41,6 +41,9 @@ func (mf *mysqlField) typeDatabaseName() string { case fieldTypeJSON: return "JSON" case fieldTypeLong: + if mf.flags&flagUnsigned != 0 { + return "UNSIGNED INT" + } return "INT" case fieldTypeLongBLOB: if mf.charSet != collations[binaryCollation] { @@ -48,6 +51,9 @@ func (mf *mysqlField) typeDatabaseName() string { } return "LONGBLOB" case fieldTypeLongLong: + if mf.flags&flagUnsigned != 0 { + return "UNSIGNED BIGINT" + } return "BIGINT" case fieldTypeMediumBLOB: if mf.charSet != collations[binaryCollation] { @@ -63,6 +69,9 @@ func (mf *mysqlField) typeDatabaseName() string { case fieldTypeSet: return "SET" case fieldTypeShort: + if mf.flags&flagUnsigned != 0 { + return "UNSIGNED SMALLINT" + } return "SMALLINT" case fieldTypeString: if mf.charSet == collations[binaryCollation] { @@ -74,6 +83,9 @@ func (mf *mysqlField) typeDatabaseName() string { case fieldTypeTimestamp: return "TIMESTAMP" case fieldTypeTiny: + if mf.flags&flagUnsigned != 0 { + return "UNSIGNED TINYINT" + } return "TINYINT" case fieldTypeTinyBLOB: if mf.charSet != collations[binaryCollation] {