From 8503ae38e9b0454dfd38df9325324eeddc7cfe96 Mon Sep 17 00:00:00 2001 From: "Adrian B.G" Date: Tue, 3 Jan 2023 16:07:17 +0200 Subject: [PATCH 1/5] Updated V7 generator to Draft04. --- generator.go | 56 +++++++++++++++++------ generator_test.go | 112 ++++++++++++++++++++++++---------------------- 2 files changed, 100 insertions(+), 68 deletions(-) diff --git a/generator.go b/generator.go index 4550bc6..b35593e 100644 --- a/generator.go +++ b/generator.go @@ -80,9 +80,9 @@ func NewV6() (UUID, error) { } // NewV7 returns a k-sortable UUID based on the current millisecond precision -// UNIX epoch and 74 bits of pseudorandom data. +// UNIX epoch and 74 bits of pseudorandom data. It supports single-node batch generation (multiple UUIDs in the same timestamp) with a Monotonic Random counter. // -// This is implemented based on revision 03 of the Peabody UUID draft, and may +// This is implemented based on revision 04 of the Peabody UUID draft, and may // be subject to change pending further revisions. Until the final specification // revision is finished, changes required to implement updates to the spec will // not be considered a breaking change. They will happen as a minor version @@ -158,7 +158,7 @@ func NewGenWithHWAF(hwaf HWAddrFunc) *Gen { func (g *Gen) NewV1() (UUID, error) { u := UUID{} - timeNow, clockSeq, err := g.getClockSequence() + timeNow, clockSeq, err := g.getClockSequence(false) if err != nil { return Nil, err } @@ -225,7 +225,7 @@ func (g *Gen) NewV6() (UUID, error) { return Nil, err } - timeNow, clockSeq, err := g.getClockSequence() + timeNow, clockSeq, err := g.getClockSequence(false) if err != nil { return Nil, err } @@ -241,8 +241,12 @@ func (g *Gen) NewV6() (UUID, error) { return u, nil } -// getClockSequence returns the epoch and clock sequence for V1 and V6 UUIDs. -func (g *Gen) getClockSequence() (uint64, uint16, error) { +// getClockSequence returns the epoch and clock sequence for V1,V6 and V7 UUIDs. +// When useUnixTS is false, it uses the Coordinated Universal Time (UTC) as a count of 100- +// +// nanosecond intervals since 00:00:00.00, 15 October 1582 (the date of +// Gregorian reform to the Christian calendar). +func (g *Gen) getClockSequence(useUnixTSMs bool) (uint64, uint16, error) { var err error g.clockSequenceOnce.Do(func() { buf := make([]byte, 2) @@ -258,7 +262,12 @@ func (g *Gen) getClockSequence() (uint64, uint16, error) { g.storageMutex.Lock() defer g.storageMutex.Unlock() - timeNow := g.getEpoch() + var timeNow uint64 + if useUnixTSMs { + timeNow = uint64(g.epochFunc().UnixMilli()) + } else { + timeNow = g.getEpoch() + } // Clock didn't change since last UUID generation. // Should increase clock sequence. if timeNow <= g.lastTime { @@ -272,28 +281,47 @@ func (g *Gen) getClockSequence() (uint64, uint16, error) { // NewV7 returns a k-sortable UUID based on the current millisecond precision // UNIX epoch and 74 bits of pseudorandom data. // -// This is implemented based on revision 03 of the Peabody UUID draft, and may +// This is implemented based on revision 04 of the Peabody UUID draft, and may // be subject to change pending further revisions. Until the final specification // revision is finished, changes required to implement updates to the spec will // not be considered a breaking change. They will happen as a minor version // releases until the spec is final. func (g *Gen) NewV7() (UUID, error) { var u UUID - - if _, err := io.ReadFull(g.rand, u[6:]); err != nil { + /* https://www.ietf.org/archive/id/draft-peabody-dispatch-new-uuid-format-04.html#name-uuid-version-7 + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | unix_ts_ms | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | unix_ts_ms | ver | rand_a | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + |var| rand_b | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | rand_b | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ */ + + ms, clockSeq, err := g.getClockSequence(true) + if err != nil { return Nil, err } - - tn := g.epochFunc() - ms := uint64(tn.Unix())*1e3 + uint64(tn.Nanosecond())/1e6 - u[0] = byte(ms >> 40) + //UUIDv7 features a 48 bit timestamp. First 32bit (4bytes) represents seconds since 1970, followed by 2 bytes for the ms granularity. + u[0] = byte(ms >> 40) //1-6 bytes: big-endian unsigned number of Unix epoch timestamp u[1] = byte(ms >> 32) u[2] = byte(ms >> 24) u[3] = byte(ms >> 16) u[4] = byte(ms >> 8) u[5] = byte(ms) + binary.BigEndian.PutUint16(u[6:8], clockSeq) // set rand_a with clock seq which is random and monotonic + //override first 4bits of u[6]. We lose the most significant bites from the clockSeq, but it is ok, we need the least significant that contains the counter to ensure the monotonic property u.SetVersion(V7) + + //set rand_b 64bits of pseudo-random bits (first 2 will be overridden) + if _, err = io.ReadFull(g.rand, u[8:16]); err != nil { + return Nil, err + } + //override first 2 bits of byte[8] for the variant u.SetVariant(VariantRFC4122) return u, nil diff --git a/generator_test.go b/generator_test.go index 01878a7..5a29775 100644 --- a/generator_test.go +++ b/generator_test.go @@ -24,7 +24,6 @@ package uuid import ( "bytes" "crypto/rand" - "encoding/binary" "errors" "fmt" "net" @@ -449,12 +448,14 @@ func testNewV6KSortable(t *testing.T) { func testNewV7(t *testing.T) { t.Run("Basic", makeTestNewV7Basic()) + t.Run("TestVector", makeTestNewV7TestVector()) t.Run("Basic10000000", makeTestNewV7Basic10000000()) t.Run("DifferentAcrossCalls", makeTestNewV7DifferentAcrossCalls()) t.Run("StaleEpoch", makeTestNewV7StaleEpoch()) t.Run("FaultyRand", makeTestNewV7FaultyRand()) t.Run("ShortRandomRead", makeTestNewV7ShortRandomRead()) t.Run("KSortable", makeTestNewV7KSortable()) + t.Run("ClockSequence", makeTestNewV7ClockSequence()) } func makeTestNewV7Basic() func(t *testing.T) { @@ -472,6 +473,38 @@ func makeTestNewV7Basic() func(t *testing.T) { } } +// makeTestNewV7TestVector as defined in Draft04 +func makeTestNewV7TestVector() func(t *testing.T) { + return func(t *testing.T) { + pRand := make([]byte, 10) + //TODO make the comparison work with + ////first 2 bytes will be read by clockSeq. First 4 bits will be overridden by Version. The next bits should be 0xCC3(3267) + //binary.LittleEndian.PutUint16(pRand[:2], uint16(0xCC3)) + ////8bytes will be read for rand_b. First 2 bits will be overridden by Variant + //binary.LittleEndian.PutUint64(pRand[2:], uint64(0x18C4DC0C0C07398F)) + + g := &Gen{ + epochFunc: func() time.Time { + return time.UnixMilli(1645557742000) + }, + rand: bytes.NewReader(pRand), + } + u, err := g.NewV7() + if err != nil { + t.Fatal(err) + } + if got, want := u.Version(), V7; got != want { + t.Errorf("got version %d, want %d", got, want) + } + if got, want := u.Variant(), VariantRFC4122; got != want { + t.Errorf("got variant %d, want %d", got, want) + } + if got, want := u.String()[:15], "017f22e2-79b0-7"; got != want { + t.Errorf("got version %q, want %q", got, want) + } + } +} + func makeTestNewV7Basic10000000() func(t *testing.T) { return func(t *testing.T) { if testing.Short() { @@ -584,61 +617,32 @@ func makeTestNewV7KSortable() func(t *testing.T) { } } -func testNewV7ClockSequence(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - } - - g := NewGen() - - // hack to try and reduce race conditions based on when the test starts - nsec := time.Now().Nanosecond() - sleepDur := int(time.Second) - nsec - time.Sleep(time.Duration(sleepDur)) - - u1, err := g.NewV7() - if err != nil { - t.Fatalf("failed to generate V7 UUID #1: %v", err) - } - - u2, err := g.NewV7() - if err != nil { - t.Fatalf("failed to generate V7 UUID #2: %v", err) - } - - time.Sleep(time.Millisecond) - - u3, err := g.NewV7() - if err != nil { - t.Fatalf("failed to generate V7 UUID #3: %v", err) - } - - time.Sleep(time.Second) - - u4, err := g.NewV7() - if err != nil { - t.Fatalf("failed to generate V7 UUID #3: %v", err) - } - - s1 := binary.BigEndian.Uint16(u1[6:8]) & 0xfff - s2 := binary.BigEndian.Uint16(u2[6:8]) & 0xfff - s3 := binary.BigEndian.Uint16(u3[6:8]) & 0xfff - s4 := binary.BigEndian.Uint16(u4[6:8]) & 0xfff - - if s1 != 0 { - t.Errorf("sequence 1 should be zero, was %d", s1) - } - - if s2 != s1+1 { - t.Errorf("sequence 2 expected to be one above sequence 1; seq 1: %d, seq 2: %d", s1, s2) - } +func makeTestNewV7ClockSequence() func(t *testing.T) { + return func(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } - if s3 != 0 { - t.Errorf("sequence 3 should be zero, was %d", s3) - } + g := NewGen() + //always return the same TS + g.epochFunc = func() time.Time { + return time.UnixMilli(1645557742000) + } + //by being KSortable with the same timestamp, it means the sequence is Not empty, and it is monotonic + uuids := make([]UUID, 10) + for i := range uuids { + u, err := g.NewV7() + testErrCheck(t, "NewV7()", "", err) + uuids[i] = u + } - if s4 != 0 { - t.Errorf("sequence 4 should be zero, was %d", s4) + for i := 1; i < len(uuids); i++ { + p, n := uuids[i-1], uuids[i] + isLess := p.String() < n.String() + if !isLess { + t.Errorf("uuids[%d] (%s) not less than uuids[%d] (%s)", i-1, p, i, n) + } + } } } From ec898399d720a7b20303f5edc0ea60a69820e7f4 Mon Sep 17 00:00:00 2001 From: "Adrian B.G" Date: Wed, 4 Jan 2023 13:48:23 +0200 Subject: [PATCH 2/5] comment fixes --- generator.go | 11 +++++++---- generator_test.go | 10 +++++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/generator.go b/generator.go index b35593e..2e5f7e3 100644 --- a/generator.go +++ b/generator.go @@ -242,10 +242,10 @@ func (g *Gen) NewV6() (UUID, error) { } // getClockSequence returns the epoch and clock sequence for V1,V6 and V7 UUIDs. -// When useUnixTS is false, it uses the Coordinated Universal Time (UTC) as a count of 100- // -// nanosecond intervals since 00:00:00.00, 15 October 1582 (the date of -// Gregorian reform to the Christian calendar). +// When useUnixTS is false, it uses the Coordinated Universal Time (UTC) as a count of 100- +// +// nanosecond intervals since 00:00:00.00, 15 October 1582 (the date of Gregorian reform to the Christian calendar). func (g *Gen) getClockSequence(useUnixTSMs bool) (uint64, uint16, error) { var err error g.clockSequenceOnce.Do(func() { @@ -312,9 +312,12 @@ func (g *Gen) NewV7() (UUID, error) { u[3] = byte(ms >> 16) u[4] = byte(ms >> 8) u[5] = byte(ms) + + //The 6th byte contains the version and partially rand_a data. + //We will lose the most significant bites from the clockSeq (with SetVersion), but it is ok, we need the least significant that contains the counter to ensure the monotonic property binary.BigEndian.PutUint16(u[6:8], clockSeq) // set rand_a with clock seq which is random and monotonic - //override first 4bits of u[6]. We lose the most significant bites from the clockSeq, but it is ok, we need the least significant that contains the counter to ensure the monotonic property + //override first 4bits of u[6]. u.SetVersion(V7) //set rand_b 64bits of pseudo-random bits (first 2 will be overridden) diff --git a/generator_test.go b/generator_test.go index 5a29775..5435898 100644 --- a/generator_test.go +++ b/generator_test.go @@ -24,6 +24,7 @@ package uuid import ( "bytes" "crypto/rand" + "encoding/binary" "errors" "fmt" "net" @@ -477,11 +478,10 @@ func makeTestNewV7Basic() func(t *testing.T) { func makeTestNewV7TestVector() func(t *testing.T) { return func(t *testing.T) { pRand := make([]byte, 10) - //TODO make the comparison work with - ////first 2 bytes will be read by clockSeq. First 4 bits will be overridden by Version. The next bits should be 0xCC3(3267) - //binary.LittleEndian.PutUint16(pRand[:2], uint16(0xCC3)) - ////8bytes will be read for rand_b. First 2 bits will be overridden by Variant - //binary.LittleEndian.PutUint64(pRand[2:], uint64(0x18C4DC0C0C07398F)) + //first 2 bytes will be read by clockSeq. First 4 bits will be overridden by Version. The next bits should be 0xCC3(3267) + binary.LittleEndian.PutUint16(pRand[:2], uint16(0xCC3)) + //8bytes will be read for rand_b. First 2 bits will be overridden by Variant + binary.LittleEndian.PutUint64(pRand[2:], uint64(0x18C4DC0C0C07398F)) g := &Gen{ epochFunc: func() time.Time { From 85bb292ef095d7075b945b4f110ea2d796ad587c Mon Sep 17 00:00:00 2001 From: "Adrian B.G" Date: Wed, 11 Jan 2023 09:28:23 +0200 Subject: [PATCH 3/5] extend test coverage for failing new rand call --- generator_test.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/generator_test.go b/generator_test.go index 5435898..996af69 100644 --- a/generator_test.go +++ b/generator_test.go @@ -573,12 +573,23 @@ func makeTestNewV7FaultyRand() func(t *testing.T) { g := &Gen{ epochFunc: time.Now, rand: &faultyReader{ - readToFail: 0, // fail immediately + readToFail: 0, }, } u, err := g.NewV7() if err == nil { - t.Errorf("got %v, nil error", u) + t.Errorf("got %v, nil error for clockSequence", u) + } + + g = &Gen{ + epochFunc: time.Now, + rand: &faultyReader{ + readToFail: 1, + }, + } + u, err = g.NewV7() + if err == nil { + t.Errorf("got %v, nil error rand_b", u) } } } From c45ce6a590b24ae7c8efc380266185bfcbdda5fa Mon Sep 17 00:00:00 2001 From: "Adrian B.G" Date: Thu, 26 Jan 2023 09:19:57 +0200 Subject: [PATCH 4/5] update readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f5db14f..4f73bec 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ This package supports the following UUID versions: * Version 5, based on SHA-1 hashing of a named value (RFC-4122) This package also supports experimental Universally Unique Identifier implementations based on a -[draft RFC](https://datatracker.ietf.org/doc/html/draft-peabody-dispatch-new-uuid-format-03) that updates RFC-4122 +[draft RFC](https://www.ietf.org/archive/id/draft-peabody-dispatch-new-uuid-format-04.html) that updates RFC-4122 * Version 6, a k-sortable id based on timestamp, and field-compatible with v1 (draft-peabody-dispatch-new-uuid-format, RFC-4122) * Version 7, a k-sortable id based on timestamp (draft-peabody-dispatch-new-uuid-format, RFC-4122) @@ -114,4 +114,4 @@ func main() { * [RFC-4122](https://tools.ietf.org/html/rfc4122) * [DCE 1.1: Authentication and Security Services](http://pubs.opengroup.org/onlinepubs/9696989899/chap5.htm#tagcjh_08_02_01_01) -* [New UUID Formats RFC Draft (Peabody) Rev 03](https://datatracker.ietf.org/doc/html/draft-peabody-dispatch-new-uuid-format-03) +* [New UUID Formats RFC Draft (Peabody) Rev 04](https://www.ietf.org/archive/id/draft-peabody-dispatch-new-uuid-format-04.html#) From 60880570649fc7f240aab6f47df7806ee95693aa Mon Sep 17 00:00:00 2001 From: "Adrian B.G" Date: Thu, 26 Jan 2023 09:33:53 +0200 Subject: [PATCH 5/5] fix more comments --- generator.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/generator.go b/generator.go index 7bf9757..44be9e1 100644 --- a/generator.go +++ b/generator.go @@ -311,7 +311,7 @@ func (g *Gen) NewV6() (UUID, error) { // getClockSequence returns the epoch and clock sequence for V1,V6 and V7 UUIDs. // -// When useUnixTS is false, it uses the Coordinated Universal Time (UTC) as a count of 100- +// When useUnixTSMs is false, it uses the Coordinated Universal Time (UTC) as a count of 100- // // nanosecond intervals since 00:00:00.00, 15 October 1582 (the date of Gregorian reform to the Christian calendar). func (g *Gen) getClockSequence(useUnixTSMs bool) (uint64, uint16, error) { @@ -381,6 +381,7 @@ func (g *Gen) NewV7() (UUID, error) { u[4] = byte(ms >> 8) u[5] = byte(ms) + //support batching by using a monotonic pseudo-random sequence //The 6th byte contains the version and partially rand_a data. //We will lose the most significant bites from the clockSeq (with SetVersion), but it is ok, we need the least significant that contains the counter to ensure the monotonic property binary.BigEndian.PutUint16(u[6:8], clockSeq) // set rand_a with clock seq which is random and monotonic