From 4e4890fc1434806fe205fd8a1ae67f538af43a58 Mon Sep 17 00:00:00 2001 From: chanxuehong Date: Sat, 30 May 2020 21:23:55 +0800 Subject: [PATCH 1/4] feat: feat: performance improvement for time format --- connection.go | 43 +++---------------- packets.go | 19 ++++----- utils.go | 51 +++++++++++++++++++++++ utils_test.go | 111 ++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 175 insertions(+), 49 deletions(-) diff --git a/connection.go b/connection.go index d1d8b29fe..907a95884 100644 --- a/connection.go +++ b/connection.go @@ -245,46 +245,13 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin if v.IsZero() { buf = append(buf, "'0000-00-00'"...) } else { - v := v.In(mc.cfg.Loc) - v = v.Add(time.Nanosecond * 500) // To round under microsecond - year := v.Year() - year100 := year / 100 - year1 := year % 100 - month := v.Month() - day := v.Day() - hour := v.Hour() - minute := v.Minute() - second := v.Second() - micro := v.Nanosecond() / 1000 - - buf = append(buf, []byte{ - '\'', - digits10[year100], digits01[year100], - digits10[year1], digits01[year1], - '-', - digits10[month], digits01[month], - '-', - digits10[day], digits01[day], - ' ', - digits10[hour], digits01[hour], - ':', - digits10[minute], digits01[minute], - ':', - digits10[second], digits01[second], - }...) - - if micro != 0 { - micro10000 := micro / 10000 - micro100 := micro / 100 % 100 - micro1 := micro % 100 - buf = append(buf, []byte{ - '.', - digits10[micro10000], digits01[micro10000], - digits10[micro100], digits01[micro100], - digits10[micro1], digits01[micro1], - }...) + b, n, err := formatDateTime(v.In(mc.cfg.Loc)) + if err != nil { + return "", err } buf = append(buf, '\'') + buf = append(buf, b[:n]...) + buf = append(buf, '\'') } case json.RawMessage: buf = append(buf, '\'') diff --git a/packets.go b/packets.go index 5cbd53298..29a3c0cd9 100644 --- a/packets.go +++ b/packets.go @@ -1110,20 +1110,17 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { paramTypes[i+i] = byte(fieldTypeString) paramTypes[i+i+1] = 0x00 - var a [64]byte - var b = a[:0] - if v.IsZero() { - b = append(b, "0000-00-00"...) + paramValues = appendLengthEncodedInteger(paramValues, uint64(len("0000-00-00"))) + paramValues = append(paramValues, "0000-00-00"...) } else { - b = v.In(mc.cfg.Loc).AppendFormat(b, timeFormat) + b, n, err := formatDateTime(v.In(mc.cfg.Loc)) + if err != nil { + return err + } + paramValues = appendLengthEncodedInteger(paramValues, uint64(n)) + paramValues = append(paramValues, b[:n]...) } - - paramValues = appendLengthEncodedInteger(paramValues, - uint64(len(b)), - ) - paramValues = append(paramValues, b...) - default: return fmt.Errorf("cannot convert type: %T", arg) } diff --git a/utils.go b/utils.go index 154ecc337..b5a763878 100644 --- a/utils.go +++ b/utils.go @@ -161,6 +161,57 @@ func parseBinaryDateTime(num uint64, data []byte, loc *time.Location) (driver.Va return nil, fmt.Errorf("invalid DATETIME packet length %d", num) } +func formatDateTime(t time.Time) (buf [32]byte, n int, err error) { + nsec := t.Nanosecond() + if nsec%1000 >= 500 { + t = t.Add(500 * time.Nanosecond) // to round under microsecond + nsec = t.Nanosecond() + } + year, month, day := t.Date() + hour, min, sec := t.Clock() + micro := nsec / 1000 + + if year < 1 || year > 9999 { + err = fmt.Errorf("invalid year: %d", year) + return + } + year100 := year / 100 + year1 := year % 100 + + buf[0], buf[1], buf[2], buf[3] = digits10[year100], digits01[year100], digits10[year1], digits01[year1] + buf[4] = '-' + buf[5], buf[6] = digits10[month], digits01[month] + buf[7] = '-' + buf[8], buf[9] = digits10[day], digits01[day] + + if hour == 0 && min == 0 && sec == 0 && micro == 0 { + n = 10 + return + } + + buf[10] = ' ' + buf[11], buf[12] = digits10[hour], digits01[hour] + buf[13] = ':' + buf[14], buf[15] = digits10[min], digits01[min] + buf[16] = ':' + buf[17], buf[18] = digits10[sec], digits01[sec] + + if micro == 0 { + n = 19 + return + } + + micro10000 := micro / 10000 + micro100 := (micro / 100) % 100 + micro1 := micro % 100 + buf[19] = '.' + buf[20], buf[21], buf[22], buf[23], buf[24], buf[25] = + digits10[micro10000], digits01[micro10000], digits10[micro100], digits01[micro100], digits10[micro1], digits01[micro1] + + n = 26 + return +} + // zeroDateTime is used in formatBinaryDateTime to avoid an allocation // if the DATE or DATETIME has the zero value. // It must never be changed. diff --git a/utils_test.go b/utils_test.go index ab29cad78..31fa12114 100644 --- a/utils_test.go +++ b/utils_test.go @@ -293,6 +293,117 @@ func TestIsolationLevelMapping(t *testing.T) { } } +func TestFormatDateTime(t *testing.T) { + tests := []struct { + t time.Time + str string + }{ + { + t: time.Date(2020, 05, 30, 0, 0, 0, 0, time.UTC), + str: "2020-05-30", + }, + { + t: time.Date(2020, 05, 30, 22, 0, 0, 0, time.UTC), + str: "2020-05-30 22:00:00", + }, + { + t: time.Date(2020, 05, 30, 22, 33, 0, 0, time.UTC), + str: "2020-05-30 22:33:00", + }, + { + t: time.Date(2020, 05, 30, 22, 33, 44, 0, time.UTC), + str: "2020-05-30 22:33:44", + }, + { + t: time.Date(2020, 05, 30, 22, 33, 44, 550000000, time.UTC), + str: "2020-05-30 22:33:44.550000", + }, + { + t: time.Date(2020, 05, 30, 22, 33, 44, 550000499, time.UTC), + str: "2020-05-30 22:33:44.550000", + }, + { + t: time.Date(2020, 05, 30, 22, 33, 44, 550000500, time.UTC), + str: "2020-05-30 22:33:44.550001", + }, + { + t: time.Date(2020, 05, 30, 22, 33, 44, 550000567, time.UTC), + str: "2020-05-30 22:33:44.550001", + }, + { + t: time.Date(2020, 05, 30, 22, 33, 44, 999999567, time.UTC), + str: "2020-05-30 22:33:45", + }, + } + for _, v := range tests { + b, n, _ := formatDateTime(v.t) + if str := string(b[:n]); str != v.str { + t.Errorf("formatDateTime(%v), have: %s, want: %s", v.t, str, v.str) + return + } + } + + // year out of range + { + v := time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC) + _, _, err := formatDateTime(v) + if err == nil { + t.Error("want an error") + return + } + } + { + v := time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC) + _, _, err := formatDateTime(v) + if err == nil { + t.Error("want an error") + return + } + } +} + +func BenchmarkGetTimeDateClockIndependent(b *testing.B) { + t := time.Now() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + t.Year() + t.Month() + t.Day() + t.Hour() + t.Minute() + t.Second() + } +} + +func BenchmarkGetTimeDateClockTogether(b *testing.B) { + t := time.Now() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + t.Date() + t.Clock() + } +} + +func BenchmarkFormatDatetime(b *testing.B) { + t := time.Now() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + formatDateTime(t) + } +} + +func BenchmarkFormatDatetimeViaStandardFormat(b *testing.B) { + t := time.Now() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + t.Format("2006-01-02 15:04:05.999999") + } +} + func TestParseDateTime(t *testing.T) { // UTC loc { From f9f72b594185fecb7eea6c5077a449928c4f651f Mon Sep 17 00:00:00 2001 From: chanxuehong Date: Mon, 1 Jun 2020 18:09:57 +0800 Subject: [PATCH 2/4] feat: rm some test functions --- utils_test.go | 42 ------------------------------------------ 1 file changed, 42 deletions(-) diff --git a/utils_test.go b/utils_test.go index 31fa12114..446d1edfc 100644 --- a/utils_test.go +++ b/utils_test.go @@ -362,48 +362,6 @@ func TestFormatDateTime(t *testing.T) { } } -func BenchmarkGetTimeDateClockIndependent(b *testing.B) { - t := time.Now() - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - t.Year() - t.Month() - t.Day() - t.Hour() - t.Minute() - t.Second() - } -} - -func BenchmarkGetTimeDateClockTogether(b *testing.B) { - t := time.Now() - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - t.Date() - t.Clock() - } -} - -func BenchmarkFormatDatetime(b *testing.B) { - t := time.Now() - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - formatDateTime(t) - } -} - -func BenchmarkFormatDatetimeViaStandardFormat(b *testing.B) { - t := time.Now() - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - t.Format("2006-01-02 15:04:05.999999") - } -} - func TestParseDateTime(t *testing.T) { // UTC loc { From 3a20c982454b5da24abffbea87c322d7f396464f Mon Sep 17 00:00:00 2001 From: chanxuehong Date: Mon, 1 Jun 2020 19:30:59 +0800 Subject: [PATCH 3/4] feat: change formatDateTime accept an input buf --- connection.go | 5 ++--- packets.go | 13 ++++++++----- utils.go | 46 ++++++++++++++++++++++------------------------ utils_test.go | 15 +++++++++------ 4 files changed, 41 insertions(+), 38 deletions(-) diff --git a/connection.go b/connection.go index 907a95884..90aec6439 100644 --- a/connection.go +++ b/connection.go @@ -245,13 +245,12 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin if v.IsZero() { buf = append(buf, "'0000-00-00'"...) } else { - b, n, err := formatDateTime(v.In(mc.cfg.Loc)) + buf = append(buf, '\'') + buf, err = appendDateTime(buf, v.In(mc.cfg.Loc)) if err != nil { return "", err } buf = append(buf, '\'') - buf = append(buf, b[:n]...) - buf = append(buf, '\'') } case json.RawMessage: buf = append(buf, '\'') diff --git a/packets.go b/packets.go index 29a3c0cd9..be51df01b 100644 --- a/packets.go +++ b/packets.go @@ -1110,17 +1110,20 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { paramTypes[i+i] = byte(fieldTypeString) paramTypes[i+i+1] = 0x00 + var a [64]byte + var b = a[:0] if v.IsZero() { - paramValues = appendLengthEncodedInteger(paramValues, uint64(len("0000-00-00"))) - paramValues = append(paramValues, "0000-00-00"...) + b = append(b, "0000-00-00"...) } else { - b, n, err := formatDateTime(v.In(mc.cfg.Loc)) + b, err = appendDateTime(b, v.In(mc.cfg.Loc)) if err != nil { return err } - paramValues = appendLengthEncodedInteger(paramValues, uint64(n)) - paramValues = append(paramValues, b[:n]...) } + paramValues = appendLengthEncodedInteger(paramValues, + uint64(len(b)), + ) + paramValues = append(paramValues, b...) default: return fmt.Errorf("cannot convert type: %T", arg) } diff --git a/utils.go b/utils.go index b5a763878..43688afe6 100644 --- a/utils.go +++ b/utils.go @@ -161,10 +161,11 @@ func parseBinaryDateTime(num uint64, data []byte, loc *time.Location) (driver.Va return nil, fmt.Errorf("invalid DATETIME packet length %d", num) } -func formatDateTime(t time.Time) (buf [32]byte, n int, err error) { +func appendDateTime(buf []byte, t time.Time) ([]byte, error) { nsec := t.Nanosecond() - if nsec%1000 >= 500 { - t = t.Add(500 * time.Nanosecond) // to round under microsecond + // to round under microsecond + if nsec%1000 >= 500 { // save half of time.Time.Add calls + t = t.Add(500 * time.Nanosecond) nsec = t.Nanosecond() } year, month, day := t.Date() @@ -172,44 +173,41 @@ func formatDateTime(t time.Time) (buf [32]byte, n int, err error) { micro := nsec / 1000 if year < 1 || year > 9999 { - err = fmt.Errorf("invalid year: %d", year) - return + return buf, errors.New("year is not in the range [1, 9999]: " + strconv.Itoa(year)) // use errors.New instead of fmt.Errorf to avoid year escape to heap } year100 := year / 100 year1 := year % 100 - buf[0], buf[1], buf[2], buf[3] = digits10[year100], digits01[year100], digits10[year1], digits01[year1] - buf[4] = '-' - buf[5], buf[6] = digits10[month], digits01[month] - buf[7] = '-' - buf[8], buf[9] = digits10[day], digits01[day] + var localBuf [26]byte // does not escape + localBuf[0], localBuf[1], localBuf[2], localBuf[3] = digits10[year100], digits01[year100], digits10[year1], digits01[year1] + localBuf[4] = '-' + localBuf[5], localBuf[6] = digits10[month], digits01[month] + localBuf[7] = '-' + localBuf[8], localBuf[9] = digits10[day], digits01[day] if hour == 0 && min == 0 && sec == 0 && micro == 0 { - n = 10 - return + return append(buf, localBuf[:10]...), nil } - buf[10] = ' ' - buf[11], buf[12] = digits10[hour], digits01[hour] - buf[13] = ':' - buf[14], buf[15] = digits10[min], digits01[min] - buf[16] = ':' - buf[17], buf[18] = digits10[sec], digits01[sec] + localBuf[10] = ' ' + localBuf[11], localBuf[12] = digits10[hour], digits01[hour] + localBuf[13] = ':' + localBuf[14], localBuf[15] = digits10[min], digits01[min] + localBuf[16] = ':' + localBuf[17], localBuf[18] = digits10[sec], digits01[sec] if micro == 0 { - n = 19 - return + return append(buf, localBuf[:19]...), nil } micro10000 := micro / 10000 micro100 := (micro / 100) % 100 micro1 := micro % 100 - buf[19] = '.' - buf[20], buf[21], buf[22], buf[23], buf[24], buf[25] = + localBuf[19] = '.' + localBuf[20], localBuf[21], localBuf[22], localBuf[23], localBuf[24], localBuf[25] = digits10[micro10000], digits01[micro10000], digits10[micro100], digits01[micro100], digits10[micro1], digits01[micro1] - n = 26 - return + return append(buf, localBuf[:]...), nil } // zeroDateTime is used in formatBinaryDateTime to avoid an allocation diff --git a/utils_test.go b/utils_test.go index 446d1edfc..f8ec844cc 100644 --- a/utils_test.go +++ b/utils_test.go @@ -293,7 +293,7 @@ func TestIsolationLevelMapping(t *testing.T) { } } -func TestFormatDateTime(t *testing.T) { +func TestAppendDateTime(t *testing.T) { tests := []struct { t time.Time str string @@ -336,9 +336,10 @@ func TestFormatDateTime(t *testing.T) { }, } for _, v := range tests { - b, n, _ := formatDateTime(v.t) - if str := string(b[:n]); str != v.str { - t.Errorf("formatDateTime(%v), have: %s, want: %s", v.t, str, v.str) + buf := make([]byte, 0, 32) + buf, _ = appendDateTime(buf, v.t) + if str := string(buf); str != v.str { + t.Errorf("appendDateTime(%v), have: %s, want: %s", v.t, str, v.str) return } } @@ -346,7 +347,8 @@ func TestFormatDateTime(t *testing.T) { // year out of range { v := time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC) - _, _, err := formatDateTime(v) + buf := make([]byte, 0, 32) + _, err := appendDateTime(buf, v) if err == nil { t.Error("want an error") return @@ -354,7 +356,8 @@ func TestFormatDateTime(t *testing.T) { } { v := time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC) - _, _, err := formatDateTime(v) + buf := make([]byte, 0, 32) + _, err := appendDateTime(buf, v) if err == nil { t.Error("want an error") return From 9b49409905757f35d8b8045b5879a95bf3ee7e13 Mon Sep 17 00:00:00 2001 From: chanxuehong Date: Mon, 1 Jun 2020 22:35:35 +0800 Subject: [PATCH 4/4] feat: restore the original blank line --- packets.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packets.go b/packets.go index 72a0387a2..6664e5ae5 100644 --- a/packets.go +++ b/packets.go @@ -1112,6 +1112,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { var a [64]byte var b = a[:0] + if v.IsZero() { b = append(b, "0000-00-00"...) } else { @@ -1120,10 +1121,12 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { return err } } + paramValues = appendLengthEncodedInteger(paramValues, uint64(len(b)), ) paramValues = append(paramValues, b...) + default: return fmt.Errorf("cannot convert type: %T", arg) }