From 82532113143a80285d9053090ed80bb2bf566c4a Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Fri, 16 Jun 2023 14:30:27 +0700 Subject: [PATCH 01/82] implements RLP encoding/decoding processing for structs --- rlp/internal/rlpstruct/rlpstruct.go | 213 ++++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 rlp/internal/rlpstruct/rlpstruct.go diff --git a/rlp/internal/rlpstruct/rlpstruct.go b/rlp/internal/rlpstruct/rlpstruct.go new file mode 100644 index 0000000000..2e3eeb6881 --- /dev/null +++ b/rlp/internal/rlpstruct/rlpstruct.go @@ -0,0 +1,213 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package rlpstruct implements struct processing for RLP encoding/decoding. +// +// In particular, this package handles all rules around field filtering, +// struct tags and nil value determination. +package rlpstruct + +import ( + "fmt" + "reflect" + "strings" +) + +// Field represents a struct field. +type Field struct { + Name string + Index int + Exported bool + Type Type + Tag string +} + +// Type represents the attributes of a Go type. +type Type struct { + Name string + Kind reflect.Kind + IsEncoder bool // whether type implements rlp.Encoder + IsDecoder bool // whether type implements rlp.Decoder + Elem *Type // non-nil for Kind values of Ptr, Slice, Array +} + +// DefaultNilValue determines whether a nil pointer to t encodes/decodes +// as an empty string or empty list. +func (t Type) DefaultNilValue() NilKind { + k := t.Kind + if isUint(k) || k == reflect.String || k == reflect.Bool || isByteArray(t) { + return NilKindString + } + return NilKindList +} + +// NilKind is the RLP value encoded in place of nil pointers. +type NilKind uint8 + +const ( + NilKindString NilKind = 0x80 + NilKindList NilKind = 0xC0 +) + +// Tags represents struct tags. +type Tags struct { + // rlp:"nil" controls whether empty input results in a nil pointer. + // nilKind is the kind of empty value allowed for the field. + NilKind NilKind + NilOK bool + + // rlp:"optional" allows for a field to be missing in the input list. + // If this is set, all subsequent fields must also be optional. + Optional bool + + // rlp:"tail" controls whether this field swallows additional list elements. It can + // only be set for the last field, which must be of slice type. + Tail bool + + // rlp:"-" ignores fields. + Ignored bool +} + +// TagError is raised for invalid struct tags. +type TagError struct { + StructType string + + // These are set by this package. + Field string + Tag string + Err string +} + +func (e TagError) Error() string { + field := "field " + e.Field + if e.StructType != "" { + field = e.StructType + "." + e.Field + } + return fmt.Sprintf("rlp: invalid struct tag %q for %s (%s)", e.Tag, field, e.Err) +} + +// ProcessFields filters the given struct fields, returning only fields +// that should be considered for encoding/decoding. +func ProcessFields(allFields []Field) ([]Field, []Tags, error) { + lastPublic := lastPublicField(allFields) + + // Gather all exported fields and their tags. + var fields []Field + var tags []Tags + for _, field := range allFields { + if !field.Exported { + continue + } + ts, err := parseTag(field, lastPublic) + if err != nil { + return nil, nil, err + } + if ts.Ignored { + continue + } + fields = append(fields, field) + tags = append(tags, ts) + } + + // Verify optional field consistency. If any optional field exists, + // all fields after it must also be optional. Note: optional + tail + // is supported. + var anyOptional bool + var firstOptionalName string + for i, ts := range tags { + name := fields[i].Name + if ts.Optional || ts.Tail { + if !anyOptional { + firstOptionalName = name + } + anyOptional = true + } else { + if anyOptional { + msg := fmt.Sprintf("must be optional because preceding field %q is optional", firstOptionalName) + return nil, nil, TagError{Field: name, Err: msg} + } + } + } + return fields, tags, nil +} + +func parseTag(field Field, lastPublic int) (Tags, error) { + name := field.Name + tag := reflect.StructTag(field.Tag) + var ts Tags + for _, t := range strings.Split(tag.Get("rlp"), ",") { + switch t = strings.TrimSpace(t); t { + case "": + // empty tag is allowed for some reason + case "-": + ts.Ignored = true + case "nil", "nilString", "nilList": + ts.NilOK = true + if field.Type.Kind != reflect.Ptr { + return ts, TagError{Field: name, Tag: t, Err: "field is not a pointer"} + } + switch t { + case "nil": + ts.NilKind = field.Type.Elem.DefaultNilValue() + case "nilString": + ts.NilKind = NilKindString + case "nilList": + ts.NilKind = NilKindList + } + case "optional": + ts.Optional = true + if ts.Tail { + return ts, TagError{Field: name, Tag: t, Err: `also has "tail" tag`} + } + case "tail": + ts.Tail = true + if field.Index != lastPublic { + return ts, TagError{Field: name, Tag: t, Err: "must be on last field"} + } + if ts.Optional { + return ts, TagError{Field: name, Tag: t, Err: `also has "optional" tag`} + } + if field.Type.Kind != reflect.Slice { + return ts, TagError{Field: name, Tag: t, Err: "field type is not slice"} + } + default: + return ts, TagError{Field: name, Tag: t, Err: "unknown tag"} + } + } + return ts, nil +} + +func lastPublicField(fields []Field) int { + last := 0 + for _, f := range fields { + if f.Exported { + last = f.Index + } + } + return last +} + +func isUint(k reflect.Kind) bool { + return k >= reflect.Uint && k <= reflect.Uintptr +} + +func isByte(typ Type) bool { + return typ.Kind == reflect.Uint8 && !typ.IsEncoder +} + +func isByteArray(typ Type) bool { + return (typ.Kind == reflect.Slice || typ.Kind == reflect.Array) && isByte(*typ.Elem) +} From fa67be8815ab694e88f71a4a31ba11a63282079c Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Fri, 16 Jun 2023 16:01:10 +0700 Subject: [PATCH 02/82] Update RLP lib --- go.mod | 64 ++--- go.sum | 171 ++++++++---- rlp/decode.go | 668 ++++++++++++++++++++++++++++------------------- rlp/doc.go | 149 ++++++++++- rlp/encbuffer.go | 423 ++++++++++++++++++++++++++++++ rlp/encode.go | 596 ++++++++++++++++-------------------------- rlp/iterator.go | 60 +++++ rlp/raw.go | 138 ++++++++++ rlp/safe.go | 27 ++ rlp/typecache.go | 268 ++++++++++++------- rlp/unsafe.go | 35 +++ 11 files changed, 1775 insertions(+), 824 deletions(-) create mode 100644 rlp/encbuffer.go create mode 100644 rlp/iterator.go create mode 100644 rlp/safe.go create mode 100644 rlp/unsafe.go diff --git a/go.mod b/go.mod index 15d820f802..e5db078ba3 100644 --- a/go.mod +++ b/go.mod @@ -4,44 +4,45 @@ go 1.19 require ( bazil.org/fuse v0.0.0-20180421153158-65cc252bf669 - github.com/VictoriaMetrics/fastcache v1.5.7 + github.com/VictoriaMetrics/fastcache v1.6.0 github.com/aristanetworks/goarista v0.0.0-20191023202215-f096da5361bb github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6 github.com/cespare/cp v1.1.1 github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea - github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf + github.com/docker/docker v1.6.2 github.com/dop251/goja v0.0.0-20230531210528-d7324b2d74f7 github.com/edsrzf/mmap-go v1.0.0 - github.com/fatih/color v1.6.0 + github.com/fatih/color v1.7.0 github.com/gizak/termui v2.2.0+incompatible github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 - github.com/go-stack/stack v1.8.0 - github.com/golang/protobuf v1.3.2 - github.com/golang/snappy v0.0.1 + github.com/go-stack/stack v1.8.1 + github.com/golang/protobuf v1.5.2 + github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb github.com/hashicorp/golang-lru v0.5.3 - github.com/huin/goupnp v1.0.0 + github.com/holiman/uint256 v1.2.2 + github.com/huin/goupnp v1.0.3 github.com/influxdata/influxdb v1.7.9 - github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 + github.com/jackpal/go-nat-pmp v1.0.2 github.com/julienschmidt/httprouter v1.3.0 github.com/karalabe/hid v1.0.0 - github.com/mattn/go-colorable v0.1.0 + github.com/mattn/go-colorable v0.1.13 github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 - github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c + github.com/olekukonko/tablewriter v0.0.5 github.com/pborman/uuid v1.2.0 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 - github.com/pkg/errors v0.8.1 + github.com/pkg/errors v0.9.1 github.com/prometheus/prometheus v1.7.2-0.20170814170113-3101606756c5 github.com/rjeczalik/notify v0.9.2 - github.com/rs/cors v1.6.0 + github.com/rs/cors v1.7.0 github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 - github.com/stretchr/testify v1.4.0 - github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d - golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 - golang.org/x/net v0.0.0-20220722155237-a158d28d115b - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 - golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f - golang.org/x/tools v0.1.12 + github.com/stretchr/testify v1.8.1 + github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 + golang.org/x/crypto v0.1.0 + golang.org/x/net v0.8.0 + golang.org/x/sync v0.1.0 + golang.org/x/sys v0.7.0 + golang.org/x/tools v0.7.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c gopkg.in/karalabe/cookiejar.v2 v2.0.0-20150724131613-8dcd6a7f4951 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce @@ -50,27 +51,28 @@ require ( ) require ( - github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect - github.com/google/go-cmp v0.3.1 // indirect github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect - github.com/google/uuid v1.0.0 // indirect - github.com/kr/pretty v0.3.0 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/maruel/panicparse v0.0.0-20160720141634-ad661195ed0e // indirect github.com/maruel/ut v1.0.2 // indirect - github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 // indirect - github.com/mattn/go-runewidth v0.0.4 // indirect + github.com/mattn/go-isatty v0.0.16 // indirect + github.com/mattn/go-runewidth v0.0.9 // indirect github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 // indirect github.com/naoina/go-stringutil v0.1.0 // indirect github.com/nsf/termbox-go v0.0.0-20170211012700-3540b76b9c77 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/rogpeppe/go-internal v1.6.1 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 // indirect - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect - golang.org/x/text v0.3.8 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - gotest.tools v2.2.0+incompatible // indirect + golang.org/x/mod v0.9.0 // indirect + golang.org/x/term v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect + golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 2fff78c90b..c913b65f5a 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Shopify/sarama v1.23.1/go.mod h1:XLH1GYJnLVE0XCr6KdJGVJRTwY30moWNJ4sERjXX6fs= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw= -github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= +github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= +github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= @@ -23,8 +23,9 @@ github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6 h1:Eey/GGQ/E5Xp1P2Ly github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU= github.com/cespare/cp v1.1.1/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic= github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -38,8 +39,8 @@ github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vs github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf h1:sh8rkQZavChcmakYiSlqu2425CHyFXLZZnvm7PDpU8M= -github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v1.6.2 h1:HlFGsy+9/xrgMmhmN+NGhCc5SHGJ7I+kHosRR1xc/aI= +github.com/docker/docker v1.6.2/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= github.com/dop251/goja v0.0.0-20230531210528-d7324b2d74f7 h1:cVGkvrdHgyBkYeB6kMCaF5j2d9Bg4trgbIpcUrKrvk4= github.com/dop251/goja v0.0.0-20230531210528-d7324b2d74f7/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= @@ -50,9 +51,12 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/fatih/color v1.6.0 h1:66qjqZk8kalYAvDRtM1AdAJQI0tj4Wrue3Eq3B3pmFU= -github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/garyburd/redigo v1.6.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/gizak/termui v2.2.0+incompatible h1:qvZU9Xll/Xd/Xr/YO+HfBKXhy8a8/94ao6vV9DSXzUE= github.com/gizak/termui v2.2.0+incompatible/go.mod h1:PkJoWUt/zacQKysNfQtcw1RW+eK2SxkieVBtl+4ovLA= @@ -63,40 +67,56 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= +github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20230207041349-798e818bf904 h1:4/hN5RUoecvl+RmJRE2YxKWtnnQls6rQjjW5oV7qg2U= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= -github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/holiman/uint256 v1.2.2 h1:TXKcSGc2WaxPD2+bmzAsVthL4+pEN0YwXcL5qED83vk= +github.com/holiman/uint256 v1.2.2/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= -github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= +github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= +github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/influxdata/influxdb v1.7.9 h1:uSeBTNO4rBkbp1Be5FKRsAmglM9nlx25TzVQRQt1An4= github.com/influxdata/influxdb v1.7.9/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/influxdata/influxdb1-client v0.0.0-20190809212627-fc22c7df067e/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= -github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= -github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= +github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -111,8 +131,9 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -123,13 +144,13 @@ github.com/maruel/panicparse v0.0.0-20160720141634-ad661195ed0e h1:e2z/lz9pvtRrE github.com/maruel/panicparse v0.0.0-20160720141634-ad661195ed0e/go.mod h1:nty42YY5QByNC5MM7q/nj938VbgPU7avs45z6NClpxI= github.com/maruel/ut v1.0.2 h1:mQTlQk3jubTbdTcza+hwoZQWhzcvE4L6K6RTtAFlA1k= github.com/maruel/ut v1.0.2/go.mod h1:RV8PwPD9dd2KFlnlCc/DB2JVvkXmyaalfc5xvmSrRSs= -github.com/mattn/go-colorable v0.1.0 h1:v2XXALHHh6zHfYTJ+cSkwtyffnaOyR1MXaA91mTrb8o= -github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 h1:USWjF42jDCSEeikX/G1g40ZWnsPXN5WkZ4jMHZWyBK4= -github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= -github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 h1:DpOJ2HYzCv8LZP15IdmG+YdwD2luVPHITV96TkirNBM= github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= @@ -144,15 +165,19 @@ github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 h1:shk/vn9oCoOTmwcou github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nsf/termbox-go v0.0.0-20170211012700-3540b76b9c77 h1:gKl78uP/I7JZ56OFtRf7nc4m1icV38hwV0In5pEGzeA= github.com/nsf/termbox-go v0.0.0-20170211012700-3540b76b9c77/go.mod h1:IuKpRQcYE1Tfu+oAQqaLisqDeXgjyyltCfsaoYN18NQ= -github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c h1:1RHs3tNxjXGHeul8z2t6H2N2TlAqpKe5yryJztRx4Jk= -github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/openconfig/gnmi v0.0.0-20190823184014-89b2bf29312c/go.mod h1:t+O9It+LKzfOAhKTT5O0ehDix+MTqbtT0T9t+7zzOvc= github.com/openconfig/reference v0.0.0-20190727015836-8dfd928c9696/go.mod h1:ym2A+zigScwkSEb/cVQB0/ZMpU3rqiH6X7WRRsxgOGw= github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= @@ -160,9 +185,10 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -181,10 +207,11 @@ github.com/prometheus/prometheus v1.7.2-0.20170814170113-3101606756c5/go.mod h1: github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rjeczalik/notify v0.9.2 h1:MiTWrPj55mNDHEiIX5YUSKefw/+lCQVoAFmD6oQm5w8= github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rs/cors v1.6.0 h1:G9tHG9lebljV9mfp9SNPDL36nCDxmo3zTlAf1YgvzmI= -github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= +github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 h1:gIlAHnH1vJb5vwEjIp5kBj/eu99p/bl0Ay2goiPe5xE= @@ -193,12 +220,16 @@ github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 h1:njlZPzLwU639 github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d h1:gZZadD8H+fF+n9CmNhYL1Y0dJB+kLOmKd7FbPJLeGHs= -github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161/go.mod h1:wM7WEvslTq+iOEAMDLSzhVuOt5BRZ05WirO+b09GHQU= github.com/templexxx/xor v0.0.0-20181023030647-4e92f724b73b/go.mod h1:5XA7W9S6mni3h5uvOC75dA3m9CCCaS83lltmc0ukdi4= github.com/tjfoc/gmsm v1.0.1/go.mod h1:XxO4hdhhrzAd+G4CjDqaOkd0hUzmtPR/d3EiBBMn/wc= @@ -210,64 +241,100 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190912141932-bc967efca4b8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190912185636-87d9f09c5d89/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/bsm/ratelimit.v1 v1.0.0-20160220154919-db14e161995a/go.mod h1:KF9sEfUPAXdG8Oev9e99iLGnl2uJMjc5B+4y3O7x610= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -275,7 +342,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= @@ -295,8 +361,11 @@ gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/rlp/decode.go b/rlp/decode.go index 60d9dab2b5..20c454ca9c 100644 --- a/rlp/decode.go +++ b/rlp/decode.go @@ -26,100 +26,78 @@ import ( "math/big" "reflect" "strings" + "sync" + + "github.com/tomochain/tomochain/rlp/internal/rlpstruct" + + "github.com/holiman/uint256" ) +//lint:ignore ST1012 EOL is not an error. + +// EOL is returned when the end of the current list +// has been reached during streaming. +var EOL = errors.New("rlp: end of list") + var ( + ErrExpectedString = errors.New("rlp: expected String or Byte") + ErrExpectedList = errors.New("rlp: expected List") + ErrCanonInt = errors.New("rlp: non-canonical integer format") + ErrCanonSize = errors.New("rlp: non-canonical size information") + ErrElemTooLarge = errors.New("rlp: element is larger than containing list") + ErrValueTooLarge = errors.New("rlp: value size exceeds available input length") + ErrMoreThanOneValue = errors.New("rlp: input contains more than one value") + + // internal errors + errNotInList = errors.New("rlp: call of ListEnd outside of any list") + errNotAtEOL = errors.New("rlp: call of ListEnd not positioned at EOL") + errUintOverflow = errors.New("rlp: uint overflow") errNoPointer = errors.New("rlp: interface given to Decode must be a pointer") errDecodeIntoNil = errors.New("rlp: pointer given to Decode must not be nil") + errUint256Large = errors.New("rlp: value too large for uint256") + + streamPool = sync.Pool{ + New: func() interface{} { return new(Stream) }, + } ) -// Decoder is implemented by types that require custom RLP -// decoding rules or need to decode into private fields. +// Decoder is implemented by types that require custom RLP decoding rules or need to decode +// into private fields. // -// The DecodeRLP method should read one value from the given -// Stream. It is not forbidden to read less or more, but it might -// be confusing. +// The DecodeRLP method should read one value from the given Stream. It is not forbidden to +// read less or more, but it might be confusing. type Decoder interface { DecodeRLP(*Stream) error } -// Decode parses RLP-encoded data from r and stores the result in the -// value pointed to by val. Val must be a non-nil pointer. If r does -// not implement ByteReader, Decode will do its own buffering. -// -// Decode uses the following type-dependent decoding rules: -// -// If the type implements the Decoder interface, decode calls -// DecodeRLP. -// -// To decode into a pointer, Decode will decode into the value pointed -// to. If the pointer is nil, a new value of the pointer's element -// type is allocated. If the pointer is non-nil, the existing value -// will be reused. -// -// To decode into a struct, Decode expects the input to be an RLP -// list. The decoded elements of the list are assigned to each public -// field in the order given by the struct's definition. The input list -// must contain an element for each decoded field. Decode returns an -// error if there are too few or too many elements. -// -// The decoding of struct fields honours certain struct tags, "tail", -// "nil" and "-". -// -// The "-" tag ignores fields. -// -// For an explanation of "tail", see the example. -// -// The "nil" tag applies to pointer-typed fields and changes the decoding -// rules for the field such that input values of size zero decode as a nil -// pointer. This tag can be useful when decoding recursive types. -// -// type StructWithEmptyOK struct { -// Foo *[20]byte `rlp:"nil"` -// } -// -// To decode into a slice, the input must be a list and the resulting -// slice will contain the input elements in order. For byte slices, -// the input must be an RLP string. Array types decode similarly, with -// the additional restriction that the number of input elements (or -// bytes) must match the array's length. -// -// To decode into a Go string, the input must be an RLP string. The -// input bytes are taken as-is and will not necessarily be valid UTF-8. -// -// To decode into an unsigned integer type, the input must also be an RLP -// string. The bytes are interpreted as a big endian representation of -// the integer. If the RLP string is larger than the bit size of the -// type, Decode will return an error. Decode also supports *big.Int. -// There is no size limit for big integers. -// -// To decode into an interface value, Decode stores one of these -// in the value: -// -// []interface{}, for RLP lists -// []byte, for RLP strings +// Decode parses RLP-encoded data from r and stores the result in the value pointed to by +// val. Please see package-level documentation for the decoding rules. Val must be a +// non-nil pointer. // -// Non-empty interface types are not supported, nor are booleans, -// signed integers, floating point numbers, maps, channels and -// functions. +// If r does not implement ByteReader, Decode will do its own buffering. // -// Note that Decode does not set an input limit for all readers -// and may be vulnerable to panics cause by huge value sizes. If -// you need an input limit, use +// Note that Decode does not set an input limit for all readers and may be vulnerable to +// panics cause by huge value sizes. If you need an input limit, use // -// NewStream(r, limit).Decode(val) +// NewStream(r, limit).Decode(val) func Decode(r io.Reader, val interface{}) error { - // TODO: this could use a Stream from a pool. - return NewStream(r, 0).Decode(val) + stream := streamPool.Get().(*Stream) + defer streamPool.Put(stream) + + stream.Reset(r, 0) + return stream.Decode(val) } -// DecodeBytes parses RLP data from b into val. -// Please see the documentation of Decode for the decoding rules. -// The input must contain exactly one value and no trailing data. +// DecodeBytes parses RLP data from b into val. Please see package-level documentation for +// the decoding rules. The input must contain exactly one value and no trailing data. func DecodeBytes(b []byte, val interface{}) error { - // TODO: this could use a Stream from a pool. r := bytes.NewReader(b) - if err := NewStream(r, uint64(len(b))).Decode(val); err != nil { + + stream := streamPool.Get().(*Stream) + defer streamPool.Put(stream) + + stream.Reset(r, uint64(len(b))) + if err := stream.Decode(val); err != nil { return err } if r.Len() > 0 { @@ -173,21 +151,26 @@ func addErrorContext(err error, ctx string) error { var ( decoderInterface = reflect.TypeOf(new(Decoder)).Elem() bigInt = reflect.TypeOf(big.Int{}) + u256Int = reflect.TypeOf(uint256.Int{}) ) -func makeDecoder(typ reflect.Type, tags tags) (dec decoder, err error) { +func makeDecoder(typ reflect.Type, tags rlpstruct.Tags) (dec decoder, err error) { kind := typ.Kind() switch { case typ == rawValueType: return decodeRawValue, nil - case typ.Implements(decoderInterface): - return decodeDecoder, nil - case kind != reflect.Ptr && reflect.PtrTo(typ).Implements(decoderInterface): - return decodeDecoderNoPtr, nil case typ.AssignableTo(reflect.PtrTo(bigInt)): return decodeBigInt, nil case typ.AssignableTo(bigInt): return decodeBigIntNoPtr, nil + case typ == reflect.PtrTo(u256Int): + return decodeU256, nil + case typ == u256Int: + return decodeU256NoPtr, nil + case kind == reflect.Ptr: + return makePtrDecoder(typ, tags) + case reflect.PtrTo(typ).Implements(decoderInterface): + return decodeDecoder, nil case isUint(kind): return decodeUint, nil case kind == reflect.Bool: @@ -198,11 +181,6 @@ func makeDecoder(typ reflect.Type, tags tags) (dec decoder, err error) { return makeListDecoder(typ, tags) case kind == reflect.Struct: return makeStructDecoder(typ) - case kind == reflect.Ptr: - if tags.nilOK { - return makeOptionalPtrDecoder(typ) - } - return makePtrDecoder(typ) case kind == reflect.Interface: return decodeInterface, nil default: @@ -252,35 +230,48 @@ func decodeBigIntNoPtr(s *Stream, val reflect.Value) error { } func decodeBigInt(s *Stream, val reflect.Value) error { - b, err := s.Bytes() + i := val.Interface().(*big.Int) + if i == nil { + i = new(big.Int) + val.Set(reflect.ValueOf(i)) + } + + err := s.decodeBigInt(i) if err != nil { return wrapStreamError(err, val.Type()) } - i := val.Interface().(*big.Int) + return nil +} + +func decodeU256NoPtr(s *Stream, val reflect.Value) error { + return decodeU256(s, val.Addr()) +} + +func decodeU256(s *Stream, val reflect.Value) error { + i := val.Interface().(*uint256.Int) if i == nil { - i = new(big.Int) + i = new(uint256.Int) val.Set(reflect.ValueOf(i)) } - // Reject leading zero bytes - if len(b) > 0 && b[0] == 0 { - return wrapStreamError(ErrCanonInt, val.Type()) + + err := s.ReadUint256(i) + if err != nil { + return wrapStreamError(err, val.Type()) } - i.SetBytes(b) return nil } -func makeListDecoder(typ reflect.Type, tag tags) (decoder, error) { +func makeListDecoder(typ reflect.Type, tag rlpstruct.Tags) (decoder, error) { etype := typ.Elem() if etype.Kind() == reflect.Uint8 && !reflect.PtrTo(etype).Implements(decoderInterface) { if typ.Kind() == reflect.Array { return decodeByteArray, nil - } else { - return decodeByteSlice, nil } + return decodeByteSlice, nil } - etypeinfo, err := cachedTypeInfo1(etype, tags{}) - if err != nil { - return nil, err + etypeinfo := theTC.infoWhileGenerating(etype, rlpstruct.Tags{}) + if etypeinfo.decoderErr != nil { + return nil, etypeinfo.decoderErr } var dec decoder switch { @@ -288,7 +279,7 @@ func makeListDecoder(typ reflect.Type, tag tags) (decoder, error) { dec = func(s *Stream, val reflect.Value) error { return decodeListArray(s, val, etypeinfo.decoder) } - case tag.tail: + case tag.Tail: // A slice with "tail" tag can occur as the last field // of a struct and is supposed to swallow all remaining // list elements. The struct decoder already called s.List, @@ -381,25 +372,23 @@ func decodeByteArray(s *Stream, val reflect.Value) error { if err != nil { return err } - vlen := val.Len() + slice := byteArrayBytes(val, val.Len()) switch kind { case Byte: - if vlen == 0 { + if len(slice) == 0 { return &decodeError{msg: "input string too long", typ: val.Type()} - } - if vlen > 1 { + } else if len(slice) > 1 { return &decodeError{msg: "input string too short", typ: val.Type()} } - bv, _ := s.Uint() - val.Index(0).SetUint(bv) + slice[0] = s.byteval + s.kind = -1 case String: - if uint64(vlen) < size { + if uint64(len(slice)) < size { return &decodeError{msg: "input string too long", typ: val.Type()} } - if uint64(vlen) > size { + if uint64(len(slice)) > size { return &decodeError{msg: "input string too short", typ: val.Type()} } - slice := val.Slice(0, vlen).Interface().([]byte) if err := s.readFull(slice); err != nil { return err } @@ -418,13 +407,25 @@ func makeStructDecoder(typ reflect.Type) (decoder, error) { if err != nil { return nil, err } + for _, f := range fields { + if f.info.decoderErr != nil { + return nil, structFieldError{typ, f.index, f.info.decoderErr} + } + } dec := func(s *Stream, val reflect.Value) (err error) { if _, err := s.List(); err != nil { return wrapStreamError(err, typ) } - for _, f := range fields { + for i, f := range fields { err := f.info.decoder(s, val.Field(f.index)) if err == EOL { + if f.optional { + // The field is optional, so reaching the end of the list before + // reaching the last field is acceptable. All remaining undecoded + // fields are zeroed. + zeroFields(val, fields[i:]) + break + } return &decodeError{msg: "too few elements", typ: typ} } else if err != nil { return addErrorContext(err, "."+typ.Field(f.index).Name) @@ -435,15 +436,29 @@ func makeStructDecoder(typ reflect.Type) (decoder, error) { return dec, nil } -// makePtrDecoder creates a decoder that decodes into -// the pointer's element type. -func makePtrDecoder(typ reflect.Type) (decoder, error) { +func zeroFields(structval reflect.Value, fields []field) { + for _, f := range fields { + fv := structval.Field(f.index) + fv.Set(reflect.Zero(fv.Type())) + } +} + +// makePtrDecoder creates a decoder that decodes into the pointer's element type. +func makePtrDecoder(typ reflect.Type, tag rlpstruct.Tags) (decoder, error) { etype := typ.Elem() - etypeinfo, err := cachedTypeInfo1(etype, tags{}) - if err != nil { - return nil, err + etypeinfo := theTC.infoWhileGenerating(etype, rlpstruct.Tags{}) + switch { + case etypeinfo.decoderErr != nil: + return nil, etypeinfo.decoderErr + case !tag.NilOK: + return makeSimplePtrDecoder(etype, etypeinfo), nil + default: + return makeNilPtrDecoder(etype, etypeinfo, tag), nil } - dec := func(s *Stream, val reflect.Value) (err error) { +} + +func makeSimplePtrDecoder(etype reflect.Type, etypeinfo *typeinfo) decoder { + return func(s *Stream, val reflect.Value) (err error) { newval := val if val.IsNil() { newval = reflect.New(etype) @@ -453,30 +468,39 @@ func makePtrDecoder(typ reflect.Type) (decoder, error) { } return err } - return dec, nil } -// makeOptionalPtrDecoder creates a decoder that decodes empty values -// as nil. Non-empty values are decoded into a value of the element type, -// just like makePtrDecoder does. +// makeNilPtrDecoder creates a decoder that decodes empty values as nil. Non-empty +// values are decoded into a value of the element type, just like makePtrDecoder does. // // This decoder is used for pointer-typed struct fields with struct tag "nil". -func makeOptionalPtrDecoder(typ reflect.Type) (decoder, error) { - etype := typ.Elem() - etypeinfo, err := cachedTypeInfo1(etype, tags{}) - if err != nil { - return nil, err - } - dec := func(s *Stream, val reflect.Value) (err error) { +func makeNilPtrDecoder(etype reflect.Type, etypeinfo *typeinfo, ts rlpstruct.Tags) decoder { + typ := reflect.PtrTo(etype) + nilPtr := reflect.Zero(typ) + + // Determine the value kind that results in nil pointer. + nilKind := typeNilKind(etype, ts) + + return func(s *Stream, val reflect.Value) (err error) { kind, size, err := s.Kind() - if err != nil || size == 0 && kind != Byte { + if err != nil { + val.Set(nilPtr) + return wrapStreamError(err, typ) + } + // Handle empty values as a nil pointer. + if kind != Byte && size == 0 { + if kind != nilKind { + return &decodeError{ + msg: fmt.Sprintf("wrong kind of empty value (got %v, want %v)", kind, nilKind), + typ: typ, + } + } // rearm s.Kind. This is important because the input // position must advance to the next value even though // we don't read anything. s.kind = -1 - // set the pointer to nil. - val.Set(reflect.Zero(typ)) - return err + val.Set(nilPtr) + return nil } newval := val if val.IsNil() { @@ -487,7 +511,6 @@ func makeOptionalPtrDecoder(typ reflect.Type) (decoder, error) { } return err } - return dec, nil } var ifsliceType = reflect.TypeOf([]interface{}{}) @@ -516,25 +539,12 @@ func decodeInterface(s *Stream, val reflect.Value) error { return nil } -// This decoder is used for non-pointer values of types -// that implement the Decoder interface using a pointer receiver. -func decodeDecoderNoPtr(s *Stream, val reflect.Value) error { - return val.Addr().Interface().(Decoder).DecodeRLP(s) -} - func decodeDecoder(s *Stream, val reflect.Value) error { - // Decoder instances are not handled using the pointer rule if the type - // implements Decoder with pointer receiver (i.e. always) - // because it might handle empty values specially. - // We need to allocate one here in this case, like makePtrDecoder does. - if val.Kind() == reflect.Ptr && val.IsNil() { - val.Set(reflect.New(val.Type().Elem())) - } - return val.Interface().(Decoder).DecodeRLP(s) + return val.Addr().Interface().(Decoder).DecodeRLP(s) } // Kind represents the kind of value contained in an RLP stream. -type Kind int +type Kind int8 const ( Byte Kind = iota @@ -555,29 +565,6 @@ func (k Kind) String() string { } } -var ( - // EOL is returned when the end of the current list - // has been reached during streaming. - EOL = errors.New("rlp: end of list") - - // Actual Errors - ErrExpectedString = errors.New("rlp: expected String or Byte") - ErrExpectedList = errors.New("rlp: expected List") - ErrCanonInt = errors.New("rlp: non-canonical integer format") - ErrCanonSize = errors.New("rlp: non-canonical size information") - ErrElemTooLarge = errors.New("rlp: element is larger than containing list") - ErrValueTooLarge = errors.New("rlp: value size exceeds available input length") - - // This error is reported by DecodeBytes if the slice contains - // additional data after the first RLP value. - ErrMoreThanOneValue = errors.New("rlp: input contains more than one value") - - // internal errors - errNotInList = errors.New("rlp: call of ListEnd outside of any list") - errNotAtEOL = errors.New("rlp: call of ListEnd not positioned at EOL") - errUintOverflow = errors.New("rlp: uint overflow") -) - // ByteReader must be implemented by any input reader for a Stream. It // is implemented by e.g. bufio.Reader and bytes.Reader. type ByteReader interface { @@ -600,22 +587,16 @@ type ByteReader interface { type Stream struct { r ByteReader - // number of bytes remaining to be read from r. - remaining uint64 - limited bool - - // auxiliary buffer for integer decoding - uintbuf []byte - - kind Kind // kind of value ahead - size uint64 // size of value ahead - byteval byte // value of single byte in type tag - kinderr error // error from last readKind - stack []listpos + remaining uint64 // number of bytes remaining to be read from r + size uint64 // size of value ahead + kinderr error // error from last readKind + stack []uint64 // list sizes + uintbuf [32]byte // auxiliary buffer for integer decoding + kind Kind // kind of value ahead + byteval byte // value of single byte in type tag + limited bool // true if input limit is in effect } -type listpos struct{ pos, size uint64 } - // NewStream creates a new decoding stream reading from r. // // If r implements the ByteReader interface, Stream will @@ -675,6 +656,37 @@ func (s *Stream) Bytes() ([]byte, error) { } } +// ReadBytes decodes the next RLP value and stores the result in b. +// The value size must match len(b) exactly. +func (s *Stream) ReadBytes(b []byte) error { + kind, size, err := s.Kind() + if err != nil { + return err + } + switch kind { + case Byte: + if len(b) != 1 { + return fmt.Errorf("input value has wrong size 1, want %d", len(b)) + } + b[0] = s.byteval + s.kind = -1 // rearm Kind + return nil + case String: + if uint64(len(b)) != size { + return fmt.Errorf("input value has wrong size %d, want %d", size, len(b)) + } + if err = s.readFull(b); err != nil { + return err + } + if size == 1 && b[0] < 128 { + return ErrCanonSize + } + return nil + default: + return ErrExpectedString + } +} + // Raw reads a raw encoded value including RLP type information. func (s *Stream) Raw() ([]byte, error) { kind, size, err := s.Kind() @@ -685,8 +697,8 @@ func (s *Stream) Raw() ([]byte, error) { s.kind = -1 // rearm Kind return []byte{s.byteval}, nil } - // the original header has already been read and is no longer - // available. read content and put a new header in front of it. + // The original header has already been read and is no longer + // available. Read content and put a new header in front of it. start := headsize(size) buf := make([]byte, uint64(start)+size) if err := s.readFull(buf[start:]); err != nil { @@ -703,10 +715,31 @@ func (s *Stream) Raw() ([]byte, error) { // Uint reads an RLP string of up to 8 bytes and returns its contents // as an unsigned integer. If the input does not contain an RLP string, the // returned error will be ErrExpectedString. +// +// Deprecated: use s.Uint64 instead. func (s *Stream) Uint() (uint64, error) { return s.uint(64) } +func (s *Stream) Uint64() (uint64, error) { + return s.uint(64) +} + +func (s *Stream) Uint32() (uint32, error) { + i, err := s.uint(32) + return uint32(i), err +} + +func (s *Stream) Uint16() (uint16, error) { + i, err := s.uint(16) + return uint16(i), err +} + +func (s *Stream) Uint8() (uint8, error) { + i, err := s.uint(8) + return uint8(i), err +} + func (s *Stream) uint(maxbits int) (uint64, error) { kind, size, err := s.Kind() if err != nil { @@ -769,7 +802,14 @@ func (s *Stream) List() (size uint64, err error) { if kind != List { return 0, ErrExpectedList } - s.stack = append(s.stack, listpos{0, size}) + + // Remove size of inner list from outer list before pushing the new size + // onto the stack. This ensures that the remaining outer list size will + // be correct after the matching call to ListEnd. + if inList, limit := s.listLimit(); inList { + s.stack[len(s.stack)-1] = limit - size + } + s.stack = append(s.stack, size) s.kind = -1 s.size = 0 return size, nil @@ -778,22 +818,116 @@ func (s *Stream) List() (size uint64, err error) { // ListEnd returns to the enclosing list. // The input reader must be positioned at the end of a list. func (s *Stream) ListEnd() error { - if len(s.stack) == 0 { + // Ensure that no more data is remaining in the current list. + if inList, listLimit := s.listLimit(); !inList { return errNotInList - } - tos := s.stack[len(s.stack)-1] - if tos.pos != tos.size { + } else if listLimit > 0 { return errNotAtEOL } s.stack = s.stack[:len(s.stack)-1] // pop - if len(s.stack) > 0 { - s.stack[len(s.stack)-1].pos += tos.size - } s.kind = -1 s.size = 0 return nil } +// MoreDataInList reports whether the current list context contains +// more data to be read. +func (s *Stream) MoreDataInList() bool { + _, listLimit := s.listLimit() + return listLimit > 0 +} + +// BigInt decodes an arbitrary-size integer value. +func (s *Stream) BigInt() (*big.Int, error) { + i := new(big.Int) + if err := s.decodeBigInt(i); err != nil { + return nil, err + } + return i, nil +} + +func (s *Stream) decodeBigInt(dst *big.Int) error { + var buffer []byte + kind, size, err := s.Kind() + switch { + case err != nil: + return err + case kind == List: + return ErrExpectedString + case kind == Byte: + buffer = s.uintbuf[:1] + buffer[0] = s.byteval + s.kind = -1 // re-arm Kind + case size == 0: + // Avoid zero-length read. + s.kind = -1 + case size <= uint64(len(s.uintbuf)): + // For integers smaller than s.uintbuf, allocating a buffer + // can be avoided. + buffer = s.uintbuf[:size] + if err := s.readFull(buffer); err != nil { + return err + } + // Reject inputs where single byte encoding should have been used. + if size == 1 && buffer[0] < 128 { + return ErrCanonSize + } + default: + // For large integers, a temporary buffer is needed. + buffer = make([]byte, size) + if err := s.readFull(buffer); err != nil { + return err + } + } + + // Reject leading zero bytes. + if len(buffer) > 0 && buffer[0] == 0 { + return ErrCanonInt + } + // Set the integer bytes. + dst.SetBytes(buffer) + return nil +} + +// ReadUint256 decodes the next value as a uint256. +func (s *Stream) ReadUint256(dst *uint256.Int) error { + var buffer []byte + kind, size, err := s.Kind() + switch { + case err != nil: + return err + case kind == List: + return ErrExpectedString + case kind == Byte: + buffer = s.uintbuf[:1] + buffer[0] = s.byteval + s.kind = -1 // re-arm Kind + case size == 0: + // Avoid zero-length read. + s.kind = -1 + case size <= uint64(len(s.uintbuf)): + // All possible uint256 values fit into s.uintbuf. + buffer = s.uintbuf[:size] + if err := s.readFull(buffer); err != nil { + return err + } + // Reject inputs where single byte encoding should have been used. + if size == 1 && buffer[0] < 128 { + return ErrCanonSize + } + default: + return errUint256Large + } + + // Reject leading zero bytes. + if len(buffer) > 0 && buffer[0] == 0 { + return ErrCanonInt + } + // Set the integer bytes. + dst.SetBytes(buffer) + return nil +} + // Decode decodes a value and stores the result in the value pointed // to by val. Please see the documentation for the Decode function // to learn about the decoding rules. @@ -809,14 +943,14 @@ func (s *Stream) Decode(val interface{}) error { if rval.IsNil() { return errDecodeIntoNil } - info, err := cachedTypeInfo(rtyp.Elem(), tags{}) + decoder, err := cachedDecoder(rtyp.Elem()) if err != nil { return err } - err = info.decoder(s, rval.Elem()) + err = decoder(s, rval.Elem()) if decErr, ok := err.(*decodeError); ok && len(decErr.ctx) > 0 { - // add decode target type to error so context has more meaning + // Add decode target type to error so context has more meaning. decErr.ctx = append(decErr.ctx, fmt.Sprint("(", rtyp.Elem(), ")")) } return err @@ -839,6 +973,9 @@ func (s *Stream) Reset(r io.Reader, inputLimit uint64) { case *bytes.Reader: s.remaining = uint64(br.Len()) s.limited = true + case *bytes.Buffer: + s.remaining = uint64(br.Len()) + s.limited = true case *strings.Reader: s.remaining = uint64(br.Len()) s.limited = true @@ -857,9 +994,8 @@ func (s *Stream) Reset(r io.Reader, inputLimit uint64) { s.size = 0 s.kind = -1 s.kinderr = nil - if s.uintbuf == nil { - s.uintbuf = make([]byte, 8) - } + s.byteval = 0 + s.uintbuf = [32]byte{} } // Kind returns the kind and size of the next value in the @@ -874,35 +1010,29 @@ func (s *Stream) Reset(r io.Reader, inputLimit uint64) { // the value. Subsequent calls to Kind (until the value is decoded) // will not advance the input reader and return cached information. func (s *Stream) Kind() (kind Kind, size uint64, err error) { - var tos *listpos - if len(s.stack) > 0 { - tos = &s.stack[len(s.stack)-1] - } - if s.kind < 0 { - s.kinderr = nil - // Don't read further if we're at the end of the - // innermost list. - if tos != nil && tos.pos == tos.size { - return 0, 0, EOL - } - s.kind, s.size, s.kinderr = s.readKind() - if s.kinderr == nil { - if tos == nil { - // At toplevel, check that the value is smaller - // than the remaining input length. - if s.limited && s.size > s.remaining { - s.kinderr = ErrValueTooLarge - } - } else { - // Inside a list, check that the value doesn't overflow the list. - if s.size > tos.size-tos.pos { - s.kinderr = ErrElemTooLarge - } - } + if s.kind >= 0 { + return s.kind, s.size, s.kinderr + } + + // Check for end of list. This needs to be done here because readKind + // checks against the list size, and would return the wrong error. + inList, listLimit := s.listLimit() + if inList && listLimit == 0 { + return 0, 0, EOL + } + // Read the actual size tag. + s.kind, s.size, s.kinderr = s.readKind() + if s.kinderr == nil { + // Check the data size of the value ahead against input limits. This + // is done here because many decoders require allocating an input + // buffer matching the value size. Checking it here protects those + // decoders from inputs declaring very large value size. + if inList && s.size > listLimit { + s.kinderr = ErrElemTooLarge + } else if s.limited && s.size > s.remaining { + s.kinderr = ErrValueTooLarge } } - // Note: this might return a sticky error generated - // by an earlier call to readKind. return s.kind, s.size, s.kinderr } @@ -929,37 +1059,35 @@ func (s *Stream) readKind() (kind Kind, size uint64, err error) { s.byteval = b return Byte, 0, nil case b < 0xB8: - // Otherwise, if a string is 0-55 bytes long, - // the RLP encoding consists of a single byte with value 0x80 plus the - // length of the string followed by the string. The range of the first - // byte is thus [0x80, 0xB7]. + // Otherwise, if a string is 0-55 bytes long, the RLP encoding consists + // of a single byte with value 0x80 plus the length of the string + // followed by the string. The range of the first byte is thus [0x80, 0xB7]. return String, uint64(b - 0x80), nil case b < 0xC0: - // If a string is more than 55 bytes long, the - // RLP encoding consists of a single byte with value 0xB7 plus the length - // of the length of the string in binary form, followed by the length of - // the string, followed by the string. For example, a length-1024 string - // would be encoded as 0xB90400 followed by the string. The range of - // the first byte is thus [0xB8, 0xBF]. + // If a string is more than 55 bytes long, the RLP encoding consists of a + // single byte with value 0xB7 plus the length of the length of the + // string in binary form, followed by the length of the string, followed + // by the string. For example, a length-1024 string would be encoded as + // 0xB90400 followed by the string. The range of the first byte is thus + // [0xB8, 0xBF]. size, err = s.readUint(b - 0xB7) if err == nil && size < 56 { err = ErrCanonSize } return String, size, err case b < 0xF8: - // If the total payload of a list - // (i.e. the combined length of all its items) is 0-55 bytes long, the - // RLP encoding consists of a single byte with value 0xC0 plus the length - // of the list followed by the concatenation of the RLP encodings of the - // items. The range of the first byte is thus [0xC0, 0xF7]. + // If the total payload of a list (i.e. the combined length of all its + // items) is 0-55 bytes long, the RLP encoding consists of a single byte + // with value 0xC0 plus the length of the list followed by the + // concatenation of the RLP encodings of the items. The range of the + // first byte is thus [0xC0, 0xF7]. return List, uint64(b - 0xC0), nil default: - // If the total payload of a list is more than 55 bytes long, - // the RLP encoding consists of a single byte with value 0xF7 - // plus the length of the length of the payload in binary - // form, followed by the length of the payload, followed by - // the concatenation of the RLP encodings of the items. The - // range of the first byte is thus [0xF8, 0xFF]. + // If the total payload of a list is more than 55 bytes long, the RLP + // encoding consists of a single byte with value 0xF7 plus the length of + // the length of the payload in binary form, followed by the length of + // the payload, followed by the concatenation of the RLP encodings of + // the items. The range of the first byte is thus [0xF8, 0xFF]. size, err = s.readUint(b - 0xF7) if err == nil && size < 56 { err = ErrCanonSize @@ -977,23 +1105,24 @@ func (s *Stream) readUint(size byte) (uint64, error) { b, err := s.readByte() return uint64(b), err default: - start := int(8 - size) - for i := 0; i < start; i++ { - s.uintbuf[i] = 0 + buffer := s.uintbuf[:8] + for i := range buffer { + buffer[i] = 0 } - if err := s.readFull(s.uintbuf[start:]); err != nil { + start := int(8 - size) + if err := s.readFull(buffer[start:]); err != nil { return 0, err } - if s.uintbuf[start] == 0 { - // Note: readUint is also used to decode integer - // values. The error needs to be adjusted to become - // ErrCanonInt in this case. + if buffer[start] == 0 { + // Note: readUint is also used to decode integer values. + // The error needs to be adjusted to become ErrCanonInt in this case. return 0, ErrCanonSize } - return binary.BigEndian.Uint64(s.uintbuf), nil + return binary.BigEndian.Uint64(buffer[:]), nil } } +// readFull reads into buf from the underlying stream. func (s *Stream) readFull(buf []byte) (err error) { if err := s.willRead(uint64(len(buf))); err != nil { return err @@ -1004,11 +1133,18 @@ func (s *Stream) readFull(buf []byte) (err error) { n += nn } if err == io.EOF { - err = io.ErrUnexpectedEOF + if n < len(buf) { + err = io.ErrUnexpectedEOF + } else { + // Readers are allowed to give EOF even though the read succeeded. + // In such cases, we discard the EOF, like io.ReadFull() does. + err = nil + } } return err } +// readByte reads a single byte from the underlying stream. func (s *Stream) readByte() (byte, error) { if err := s.willRead(1); err != nil { return 0, err @@ -1020,16 +1156,16 @@ func (s *Stream) readByte() (byte, error) { return b, err } +// willRead is called before any read from the underlying stream. It checks +// n against size limits, and updates the limits if n doesn't overflow them. func (s *Stream) willRead(n uint64) error { s.kind = -1 // rearm Kind - if len(s.stack) > 0 { - // check list overflow - tos := s.stack[len(s.stack)-1] - if n > tos.size-tos.pos { + if inList, limit := s.listLimit(); inList { + if n > limit { return ErrElemTooLarge } - s.stack[len(s.stack)-1].pos += n + s.stack[len(s.stack)-1] = limit - n } if s.limited { if n > s.remaining { @@ -1039,3 +1175,11 @@ func (s *Stream) willRead(n uint64) error { } return nil } + +// listLimit returns the amount of data remaining in the innermost list. +func (s *Stream) listLimit() (inList bool, limit uint64) { + if len(s.stack) == 0 { + return false, 0 + } + return true, s.stack[len(s.stack)-1] +} diff --git a/rlp/doc.go b/rlp/doc.go index b3a81fe232..eeeee9a43a 100644 --- a/rlp/doc.go +++ b/rlp/doc.go @@ -17,17 +17,142 @@ /* Package rlp implements the RLP serialization format. -The purpose of RLP (Recursive Linear Prefix) is to encode arbitrarily -nested arrays of binary data, and RLP is the main encoding method used -to serialize objects in Ethereum. The only purpose of RLP is to encode -structure; encoding specific atomic data types (eg. strings, ints, -floats) is left up to higher-order protocols; in Ethereum integers -must be represented in big endian binary form with no leading zeroes -(thus making the integer value zero equivalent to the empty byte -array). - -RLP values are distinguished by a type tag. The type tag precedes the -value in the input stream and defines the size and kind of the bytes -that follow. +The purpose of RLP (Recursive Linear Prefix) is to encode arbitrarily nested arrays of +binary data, and RLP is the main encoding method used to serialize objects in Ethereum. +The only purpose of RLP is to encode structure; encoding specific atomic data types (eg. +strings, ints, floats) is left up to higher-order protocols. In Ethereum integers must be +represented in big endian binary form with no leading zeroes (thus making the integer +value zero equivalent to the empty string). + +RLP values are distinguished by a type tag. The type tag precedes the value in the input +stream and defines the size and kind of the bytes that follow. + +# Encoding Rules + +Package rlp uses reflection and encodes RLP based on the Go type of the value. + +If the type implements the Encoder interface, Encode calls EncodeRLP. It does not +call EncodeRLP on nil pointer values. + +To encode a pointer, the value being pointed to is encoded. A nil pointer to a struct +type, slice or array always encodes as an empty RLP list unless the slice or array has +element type byte. A nil pointer to any other value encodes as the empty string. + +Struct values are encoded as an RLP list of all their encoded public fields. Recursive +struct types are supported. + +To encode slices and arrays, the elements are encoded as an RLP list of the value's +elements. Note that arrays and slices with element type uint8 or byte are always encoded +as an RLP string. + +A Go string is encoded as an RLP string. + +An unsigned integer value is encoded as an RLP string. Zero always encodes as an empty RLP +string. big.Int values are treated as integers. Signed integers (int, int8, int16, ...) +are not supported and will return an error when encoding. + +Boolean values are encoded as the unsigned integers zero (false) and one (true). + +An interface value encodes as the value contained in the interface. + +Floating point numbers, maps, channels and functions are not supported. + +# Decoding Rules + +Decoding uses the following type-dependent rules: + +If the type implements the Decoder interface, DecodeRLP is called. + +To decode into a pointer, the value will be decoded as the element type of the pointer. If +the pointer is nil, a new value of the pointer's element type is allocated. If the pointer +is non-nil, the existing value will be reused. Note that package rlp never leaves a +pointer-type struct field as nil unless one of the "nil" struct tags is present. + +To decode into a struct, decoding expects the input to be an RLP list. The decoded +elements of the list are assigned to each public field in the order given by the struct's +definition. The input list must contain an element for each decoded field. Decoding +returns an error if there are too few or too many elements for the struct. + +To decode into a slice, the input must be a list and the resulting slice will contain the +input elements in order. For byte slices, the input must be an RLP string. Array types +decode similarly, with the additional restriction that the number of input elements (or +bytes) must match the array's defined length. + +To decode into a Go string, the input must be an RLP string. The input bytes are taken +as-is and will not necessarily be valid UTF-8. + +To decode into an unsigned integer type, the input must also be an RLP string. The bytes +are interpreted as a big endian representation of the integer. If the RLP string is larger +than the bit size of the type, decoding will return an error. Decode also supports +*big.Int. There is no size limit for big integers. + +To decode into a boolean, the input must contain an unsigned integer of value zero (false) +or one (true). + +To decode into an interface value, one of these types is stored in the value: + + []interface{}, for RLP lists + []byte, for RLP strings + +Non-empty interface types are not supported when decoding. +Signed integers, floating point numbers, maps, channels and functions cannot be decoded into. + +# Struct Tags + +As with other encoding packages, the "-" tag ignores fields. + + type StructWithIgnoredField struct{ + Ignored uint `rlp:"-"` + Field uint + } + +Go struct values encode/decode as RLP lists. There are two ways of influencing the mapping +of fields to list elements. The "tail" tag, which may only be used on the last exported +struct field, allows slurping up any excess list elements into a slice. + + type StructWithTail struct{ + Field uint + Tail []string `rlp:"tail"` + } + +The "optional" tag says that the field may be omitted if it is zero-valued. If this tag is +used on a struct field, all subsequent public fields must also be declared optional. + +When encoding a struct with optional fields, the output RLP list contains all values up to +the last non-zero optional field. + +When decoding into a struct, optional fields may be omitted from the end of the input +list. For the example below, this means input lists of one, two, or three elements are +accepted. + + type StructWithOptionalFields struct{ + Required uint + Optional1 uint `rlp:"optional"` + Optional2 uint `rlp:"optional"` + } + +The "nil", "nilList" and "nilString" tags apply to pointer-typed fields only, and change +the decoding rules for the field type. For regular pointer fields without the "nil" tag, +input values must always match the required input length exactly and the decoder does not +produce nil values. When the "nil" tag is set, input values of size zero decode as a nil +pointer. This is especially useful for recursive types. + + type StructWithNilField struct { + Field *[3]byte `rlp:"nil"` + } + +In the example above, Field allows two possible input sizes. For input 0xC180 (a list +containing an empty string) Field is set to nil after decoding. For input 0xC483000000 (a +list containing a 3-byte string), Field is set to a non-nil array pointer. + +RLP supports two kinds of empty values: empty lists and empty strings. When using the +"nil" tag, the kind of empty value allowed for a type is chosen automatically. A field +whose Go type is a pointer to an unsigned integer, string, boolean or byte array/slice +expects an empty RLP string. Any other pointer field type encodes/decodes as an empty RLP +list. + +The choice of null value can be made explicit with the "nilList" and "nilString" struct +tags. Using these tags encodes/decodes a Go nil pointer value as the empty RLP value kind +defined by the tag. */ package rlp diff --git a/rlp/encbuffer.go b/rlp/encbuffer.go new file mode 100644 index 0000000000..8d3a3b2293 --- /dev/null +++ b/rlp/encbuffer.go @@ -0,0 +1,423 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rlp + +import ( + "encoding/binary" + "io" + "math/big" + "reflect" + "sync" + + "github.com/holiman/uint256" +) + +type encBuffer struct { + str []byte // string data, contains everything except list headers + lheads []listhead // all list headers + lhsize int // sum of sizes of all encoded list headers + sizebuf [9]byte // auxiliary buffer for uint encoding +} + +// The global encBuffer pool. +var encBufferPool = sync.Pool{ + New: func() interface{} { return new(encBuffer) }, +} + +func getEncBuffer() *encBuffer { + buf := encBufferPool.Get().(*encBuffer) + buf.reset() + return buf +} + +func (buf *encBuffer) reset() { + buf.lhsize = 0 + buf.str = buf.str[:0] + buf.lheads = buf.lheads[:0] +} + +// size returns the length of the encoded data. +func (buf *encBuffer) size() int { + return len(buf.str) + buf.lhsize +} + +// makeBytes creates the encoder output. +func (buf *encBuffer) makeBytes() []byte { + out := make([]byte, buf.size()) + buf.copyTo(out) + return out +} + +func (buf *encBuffer) copyTo(dst []byte) { + strpos := 0 + pos := 0 + for _, head := range buf.lheads { + // write string data before header + n := copy(dst[pos:], buf.str[strpos:head.offset]) + pos += n + strpos += n + // write the header + enc := head.encode(dst[pos:]) + pos += len(enc) + } + // copy string data after the last list header + copy(dst[pos:], buf.str[strpos:]) +} + +// writeTo writes the encoder output to w. +func (buf *encBuffer) writeTo(w io.Writer) (err error) { + strpos := 0 + for _, head := range buf.lheads { + // write string data before header + if head.offset-strpos > 0 { + n, err := w.Write(buf.str[strpos:head.offset]) + strpos += n + if err != nil { + return err + } + } + // write the header + enc := head.encode(buf.sizebuf[:]) + if _, err = w.Write(enc); err != nil { + return err + } + } + if strpos < len(buf.str) { + // write string data after the last list header + _, err = w.Write(buf.str[strpos:]) + } + return err +} + +// Write implements io.Writer and appends b directly to the output. +func (buf *encBuffer) Write(b []byte) (int, error) { + buf.str = append(buf.str, b...) + return len(b), nil +} + +// writeBool writes b as the integer 0 (false) or 1 (true). +func (buf *encBuffer) writeBool(b bool) { + if b { + buf.str = append(buf.str, 0x01) + } else { + buf.str = append(buf.str, 0x80) + } +} + +func (buf *encBuffer) writeUint64(i uint64) { + if i == 0 { + buf.str = append(buf.str, 0x80) + } else if i < 128 { + // fits single byte + buf.str = append(buf.str, byte(i)) + } else { + s := putint(buf.sizebuf[1:], i) + buf.sizebuf[0] = 0x80 + byte(s) + buf.str = append(buf.str, buf.sizebuf[:s+1]...) + } +} + +func (buf *encBuffer) writeBytes(b []byte) { + if len(b) == 1 && b[0] <= 0x7F { + // fits single byte, no string header + buf.str = append(buf.str, b[0]) + } else { + buf.encodeStringHeader(len(b)) + buf.str = append(buf.str, b...) + } +} + +func (buf *encBuffer) writeString(s string) { + buf.writeBytes([]byte(s)) +} + +// wordBytes is the number of bytes in a big.Word +const wordBytes = (32 << (uint64(^big.Word(0)) >> 63)) / 8 + +// writeBigInt writes i as an integer. +func (buf *encBuffer) writeBigInt(i *big.Int) { + bitlen := i.BitLen() + if bitlen <= 64 { + buf.writeUint64(i.Uint64()) + return + } + // Integer is larger than 64 bits, encode from i.Bits(). + // The minimal byte length is bitlen rounded up to the next + // multiple of 8, divided by 8. + length := ((bitlen + 7) & -8) >> 3 + buf.encodeStringHeader(length) + buf.str = append(buf.str, make([]byte, length)...) + index := length + bytesBuf := buf.str[len(buf.str)-length:] + for _, d := range i.Bits() { + for j := 0; j < wordBytes && index > 0; j++ { + index-- + bytesBuf[index] = byte(d) + d >>= 8 + } + } +} + +// writeUint256 writes z as an integer. +func (buf *encBuffer) writeUint256(z *uint256.Int) { + bitlen := z.BitLen() + if bitlen <= 64 { + buf.writeUint64(z.Uint64()) + return + } + nBytes := byte((bitlen + 7) / 8) + var b [33]byte + binary.BigEndian.PutUint64(b[1:9], z[3]) + binary.BigEndian.PutUint64(b[9:17], z[2]) + binary.BigEndian.PutUint64(b[17:25], z[1]) + binary.BigEndian.PutUint64(b[25:33], z[0]) + b[32-nBytes] = 0x80 + nBytes + buf.str = append(buf.str, b[32-nBytes:]...) +} + +// list adds a new list header to the header stack. It returns the index of the header. +// Call listEnd with this index after encoding the content of the list. +func (buf *encBuffer) list() int { + buf.lheads = append(buf.lheads, listhead{offset: len(buf.str), size: buf.lhsize}) + return len(buf.lheads) - 1 +} + +func (buf *encBuffer) listEnd(index int) { + lh := &buf.lheads[index] + lh.size = buf.size() - lh.offset - lh.size + if lh.size < 56 { + buf.lhsize++ // length encoded into kind tag + } else { + buf.lhsize += 1 + intsize(uint64(lh.size)) + } +} + +func (buf *encBuffer) encode(val interface{}) error { + rval := reflect.ValueOf(val) + writer, err := cachedWriter(rval.Type()) + if err != nil { + return err + } + return writer(rval, buf) +} + +func (buf *encBuffer) encodeStringHeader(size int) { + if size < 56 { + buf.str = append(buf.str, 0x80+byte(size)) + } else { + sizesize := putint(buf.sizebuf[1:], uint64(size)) + buf.sizebuf[0] = 0xB7 + byte(sizesize) + buf.str = append(buf.str, buf.sizebuf[:sizesize+1]...) + } +} + +// encReader is the io.Reader returned by EncodeToReader. +// It releases its encbuf at EOF. +type encReader struct { + buf *encBuffer // the buffer we're reading from. this is nil when we're at EOF. + lhpos int // index of list header that we're reading + strpos int // current position in string buffer + piece []byte // next piece to be read +} + +func (r *encReader) Read(b []byte) (n int, err error) { + for { + if r.piece = r.next(); r.piece == nil { + // Put the encode buffer back into the pool at EOF when it + // is first encountered. Subsequent calls still return EOF + // as the error but the buffer is no longer valid. + if r.buf != nil { + encBufferPool.Put(r.buf) + r.buf = nil + } + return n, io.EOF + } + nn := copy(b[n:], r.piece) + n += nn + if nn < len(r.piece) { + // piece didn't fit, see you next time. + r.piece = r.piece[nn:] + return n, nil + } + r.piece = nil + } +} + +// next returns the next piece of data to be read. +// it returns nil at EOF. +func (r *encReader) next() []byte { + switch { + case r.buf == nil: + return nil + + case r.piece != nil: + // There is still data available for reading. + return r.piece + + case r.lhpos < len(r.buf.lheads): + // We're before the last list header. + head := r.buf.lheads[r.lhpos] + sizebefore := head.offset - r.strpos + if sizebefore > 0 { + // String data before header. + p := r.buf.str[r.strpos:head.offset] + r.strpos += sizebefore + return p + } + r.lhpos++ + return head.encode(r.buf.sizebuf[:]) + + case r.strpos < len(r.buf.str): + // String data at the end, after all list headers. + p := r.buf.str[r.strpos:] + r.strpos = len(r.buf.str) + return p + + default: + return nil + } +} + +func encBufferFromWriter(w io.Writer) *encBuffer { + switch w := w.(type) { + case EncoderBuffer: + return w.buf + case *EncoderBuffer: + return w.buf + case *encBuffer: + return w + default: + return nil + } +} + +// EncoderBuffer is a buffer for incremental encoding. +// +// The zero value is NOT ready for use. To get a usable buffer, +// create it using NewEncoderBuffer or call Reset. +type EncoderBuffer struct { + buf *encBuffer + dst io.Writer + + ownBuffer bool +} + +// NewEncoderBuffer creates an encoder buffer. +func NewEncoderBuffer(dst io.Writer) EncoderBuffer { + var w EncoderBuffer + w.Reset(dst) + return w +} + +// Reset truncates the buffer and sets the output destination. +func (w *EncoderBuffer) Reset(dst io.Writer) { + if w.buf != nil && !w.ownBuffer { + panic("can't Reset derived EncoderBuffer") + } + + // If the destination writer has an *encBuffer, use it. + // Note that w.ownBuffer is left false here. + if dst != nil { + if outer := encBufferFromWriter(dst); outer != nil { + *w = EncoderBuffer{outer, nil, false} + return + } + } + + // Get a fresh buffer. + if w.buf == nil { + w.buf = encBufferPool.Get().(*encBuffer) + w.ownBuffer = true + } + w.buf.reset() + w.dst = dst +} + +// Flush writes encoded RLP data to the output writer. This can only be called once. +// If you want to re-use the buffer after Flush, you must call Reset. +func (w *EncoderBuffer) Flush() error { + var err error + if w.dst != nil { + err = w.buf.writeTo(w.dst) + } + // Release the internal buffer. + if w.ownBuffer { + encBufferPool.Put(w.buf) + } + *w = EncoderBuffer{} + return err +} + +// ToBytes returns the encoded bytes. +func (w *EncoderBuffer) ToBytes() []byte { + return w.buf.makeBytes() +} + +// AppendToBytes appends the encoded bytes to dst. +func (w *EncoderBuffer) AppendToBytes(dst []byte) []byte { + size := w.buf.size() + out := append(dst, make([]byte, size)...) + w.buf.copyTo(out[len(dst):]) + return out +} + +// Write appends b directly to the encoder output. +func (w EncoderBuffer) Write(b []byte) (int, error) { + return w.buf.Write(b) +} + +// WriteBool writes b as the integer 0 (false) or 1 (true). +func (w EncoderBuffer) WriteBool(b bool) { + w.buf.writeBool(b) +} + +// WriteUint64 encodes an unsigned integer. +func (w EncoderBuffer) WriteUint64(i uint64) { + w.buf.writeUint64(i) +} + +// WriteBigInt encodes a big.Int as an RLP string. +// Note: Unlike with Encode, the sign of i is ignored. +func (w EncoderBuffer) WriteBigInt(i *big.Int) { + w.buf.writeBigInt(i) +} + +// WriteUint256 encodes uint256.Int as an RLP string. +func (w EncoderBuffer) WriteUint256(i *uint256.Int) { + w.buf.writeUint256(i) +} + +// WriteBytes encodes b as an RLP string. +func (w EncoderBuffer) WriteBytes(b []byte) { + w.buf.writeBytes(b) +} + +// WriteString encodes s as an RLP string. +func (w EncoderBuffer) WriteString(s string) { + w.buf.writeString(s) +} + +// List starts a list. It returns an internal index. Call EndList with +// this index after encoding the content to finish the list. +func (w EncoderBuffer) List() int { + return w.buf.list() +} + +// ListEnd finishes the given list. +func (w EncoderBuffer) ListEnd(index int) { + w.buf.listEnd(index) +} diff --git a/rlp/encode.go b/rlp/encode.go index 44592c2f53..f34be7f3df 100644 --- a/rlp/encode.go +++ b/rlp/encode.go @@ -17,20 +17,29 @@ package rlp import ( + "errors" "fmt" "io" "math/big" "reflect" - "sync" + + "github.com/tomochain/tomochain/rlp/internal/rlpstruct" + + "github.com/holiman/uint256" ) var ( // Common encoded values. // These are useful when implementing EncodeRLP. + + // EmptyString is the encoding of an empty string. EmptyString = []byte{0x80} - EmptyList = []byte{0xC0} + // EmptyList is the encoding of an empty list. + EmptyList = []byte{0xC0} ) +var ErrNegativeBigInt = errors.New("rlp: cannot encode negative big.Int") + // Encoder is implemented by types that require custom // encoding rules or want to encode private fields. type Encoder interface { @@ -49,80 +58,48 @@ type Encoder interface { // perform many small writes in some cases. Consider making w // buffered. // -// Encode uses the following type-dependent encoding rules: -// -// If the type implements the Encoder interface, Encode calls -// EncodeRLP. This is true even for nil pointers, please see the -// documentation for Encoder. -// -// To encode a pointer, the value being pointed to is encoded. For nil -// pointers, Encode will encode the zero value of the type. A nil -// pointer to a struct type always encodes as an empty RLP list. -// A nil pointer to an array encodes as an empty list (or empty string -// if the array has element type byte). -// -// Struct values are encoded as an RLP list of all their encoded -// public fields. Recursive struct types are supported. -// -// To encode slices and arrays, the elements are encoded as an RLP -// list of the value's elements. Note that arrays and slices with -// element type uint8 or byte are always encoded as an RLP string. -// -// A Go string is encoded as an RLP string. -// -// An unsigned integer value is encoded as an RLP string. Zero always -// encodes as an empty RLP string. Encode also supports *big.Int. -// -// An interface value encodes as the value contained in the interface. -// -// Boolean values are not supported, nor are signed integers, floating -// point numbers, maps, channels and functions. +// Please see package-level documentation of encoding rules. func Encode(w io.Writer, val interface{}) error { - if outer, ok := w.(*encbuf); ok { - // Encode was called by some type's EncodeRLP. - // Avoid copying by writing to the outer encbuf directly. - return outer.encode(val) + // Optimization: reuse *encBuffer when called by EncodeRLP. + if buf := encBufferFromWriter(w); buf != nil { + return buf.encode(val) } - eb := encbufPool.Get().(*encbuf) - defer encbufPool.Put(eb) - eb.reset() - if err := eb.encode(val); err != nil { + + buf := getEncBuffer() + defer encBufferPool.Put(buf) + if err := buf.encode(val); err != nil { return err } - return eb.toWriter(w) + return buf.writeTo(w) } -// EncodeBytes returns the RLP encoding of val. -// Please see the documentation of Encode for the encoding rules. +// EncodeToBytes returns the RLP encoding of val. +// Please see package-level documentation for the encoding rules. func EncodeToBytes(val interface{}) ([]byte, error) { - eb := encbufPool.Get().(*encbuf) - defer encbufPool.Put(eb) - eb.reset() - if err := eb.encode(val); err != nil { + buf := getEncBuffer() + defer encBufferPool.Put(buf) + + if err := buf.encode(val); err != nil { return nil, err } - return eb.toBytes(), nil + return buf.makeBytes(), nil } -// EncodeReader returns a reader from which the RLP encoding of val +// EncodeToReader returns a reader from which the RLP encoding of val // can be read. The returned size is the total size of the encoded // data. // // Please see the documentation of Encode for the encoding rules. func EncodeToReader(val interface{}) (size int, r io.Reader, err error) { - eb := encbufPool.Get().(*encbuf) - eb.reset() - if err := eb.encode(val); err != nil { + buf := getEncBuffer() + if err := buf.encode(val); err != nil { + encBufferPool.Put(buf) return 0, nil, err } - return eb.size(), &encReader{buf: eb}, nil -} - -type encbuf struct { - str []byte // string data, contains everything except list headers - lheads []*listhead // all list headers - lhsize int // sum of sizes of all encoded list headers - sizebuf []byte // 9-byte auxiliary buffer for uint encoding + // Note: can't put the reader back into the pool here + // because it is held by encReader. The reader puts it + // back when it has been fully consumed. + return buf.size(), &encReader{buf: buf}, nil } type listhead struct { @@ -151,214 +128,32 @@ func puthead(buf []byte, smalltag, largetag byte, size uint64) int { if size < 56 { buf[0] = smalltag + byte(size) return 1 - } else { - sizesize := putint(buf[1:], size) - buf[0] = largetag + byte(sizesize) - return sizesize + 1 - } -} - -// encbufs are pooled. -var encbufPool = sync.Pool{ - New: func() interface{} { return &encbuf{sizebuf: make([]byte, 9)} }, -} - -func (w *encbuf) reset() { - w.lhsize = 0 - if w.str != nil { - w.str = w.str[:0] - } - if w.lheads != nil { - w.lheads = w.lheads[:0] - } -} - -// encbuf implements io.Writer so it can be passed it into EncodeRLP. -func (w *encbuf) Write(b []byte) (int, error) { - w.str = append(w.str, b...) - return len(b), nil -} - -func (w *encbuf) encode(val interface{}) error { - rval := reflect.ValueOf(val) - ti, err := cachedTypeInfo(rval.Type(), tags{}) - if err != nil { - return err - } - return ti.writer(rval, w) -} - -func (w *encbuf) encodeStringHeader(size int) { - if size < 56 { - w.str = append(w.str, 0x80+byte(size)) - } else { - // TODO: encode to w.str directly - sizesize := putint(w.sizebuf[1:], uint64(size)) - w.sizebuf[0] = 0xB7 + byte(sizesize) - w.str = append(w.str, w.sizebuf[:sizesize+1]...) - } -} - -func (w *encbuf) encodeString(b []byte) { - if len(b) == 1 && b[0] <= 0x7F { - // fits single byte, no string header - w.str = append(w.str, b[0]) - } else { - w.encodeStringHeader(len(b)) - w.str = append(w.str, b...) - } -} - -func (w *encbuf) list() *listhead { - lh := &listhead{offset: len(w.str), size: w.lhsize} - w.lheads = append(w.lheads, lh) - return lh -} - -func (w *encbuf) listEnd(lh *listhead) { - lh.size = w.size() - lh.offset - lh.size - if lh.size < 56 { - w.lhsize += 1 // length encoded into kind tag - } else { - w.lhsize += 1 + intsize(uint64(lh.size)) - } -} - -func (w *encbuf) size() int { - return len(w.str) + w.lhsize -} - -func (w *encbuf) toBytes() []byte { - out := make([]byte, w.size()) - strpos := 0 - pos := 0 - for _, head := range w.lheads { - // write string data before header - n := copy(out[pos:], w.str[strpos:head.offset]) - pos += n - strpos += n - // write the header - enc := head.encode(out[pos:]) - pos += len(enc) } - // copy string data after the last list header - copy(out[pos:], w.str[strpos:]) - return out + sizesize := putint(buf[1:], size) + buf[0] = largetag + byte(sizesize) + return sizesize + 1 } -func (w *encbuf) toWriter(out io.Writer) (err error) { - strpos := 0 - for _, head := range w.lheads { - // write string data before header - if head.offset-strpos > 0 { - n, err := out.Write(w.str[strpos:head.offset]) - strpos += n - if err != nil { - return err - } - } - // write the header - enc := head.encode(w.sizebuf) - if _, err = out.Write(enc); err != nil { - return err - } - } - if strpos < len(w.str) { - // write string data after the last list header - _, err = out.Write(w.str[strpos:]) - } - return err -} - -// encReader is the io.Reader returned by EncodeToReader. -// It releases its encbuf at EOF. -type encReader struct { - buf *encbuf // the buffer we're reading from. this is nil when we're at EOF. - lhpos int // index of list header that we're reading - strpos int // current position in string buffer - piece []byte // next piece to be read -} - -func (r *encReader) Read(b []byte) (n int, err error) { - for { - if r.piece = r.next(); r.piece == nil { - // Put the encode buffer back into the pool at EOF when it - // is first encountered. Subsequent calls still return EOF - // as the error but the buffer is no longer valid. - if r.buf != nil { - encbufPool.Put(r.buf) - r.buf = nil - } - return n, io.EOF - } - nn := copy(b[n:], r.piece) - n += nn - if nn < len(r.piece) { - // piece didn't fit, see you next time. - r.piece = r.piece[nn:] - return n, nil - } - r.piece = nil - } -} - -// next returns the next piece of data to be read. -// it returns nil at EOF. -func (r *encReader) next() []byte { - switch { - case r.buf == nil: - return nil - - case r.piece != nil: - // There is still data available for reading. - return r.piece - - case r.lhpos < len(r.buf.lheads): - // We're before the last list header. - head := r.buf.lheads[r.lhpos] - sizebefore := head.offset - r.strpos - if sizebefore > 0 { - // String data before header. - p := r.buf.str[r.strpos:head.offset] - r.strpos += sizebefore - return p - } else { - r.lhpos++ - return head.encode(r.buf.sizebuf) - } - - case r.strpos < len(r.buf.str): - // String data at the end, after all list headers. - p := r.buf.str[r.strpos:] - r.strpos = len(r.buf.str) - return p - - default: - return nil - } -} - -var ( - encoderInterface = reflect.TypeOf(new(Encoder)).Elem() - big0 = big.NewInt(0) -) +var encoderInterface = reflect.TypeOf(new(Encoder)).Elem() // makeWriter creates a writer function for the given type. -func makeWriter(typ reflect.Type, ts tags) (writer, error) { +func makeWriter(typ reflect.Type, ts rlpstruct.Tags) (writer, error) { kind := typ.Kind() switch { case typ == rawValueType: return writeRawValue, nil - case typ.Implements(encoderInterface): - return writeEncoder, nil - case kind != reflect.Ptr && reflect.PtrTo(typ).Implements(encoderInterface): - return writeEncoderNoPtr, nil - case kind == reflect.Interface: - return writeInterface, nil case typ.AssignableTo(reflect.PtrTo(bigInt)): return writeBigIntPtr, nil case typ.AssignableTo(bigInt): return writeBigIntNoPtr, nil + case typ == reflect.PtrTo(u256Int): + return writeU256IntPtr, nil + case typ == u256Int: + return writeU256IntNoPtr, nil + case kind == reflect.Ptr: + return makePtrWriter(typ, ts) + case reflect.PtrTo(typ).Implements(encoderInterface): + return makeEncoderWriter(typ), nil case isUint(kind): return writeUint, nil case kind == reflect.Bool: @@ -368,97 +163,116 @@ func makeWriter(typ reflect.Type, ts tags) (writer, error) { case kind == reflect.Slice && isByte(typ.Elem()): return writeBytes, nil case kind == reflect.Array && isByte(typ.Elem()): - return writeByteArray, nil + return makeByteArrayWriter(typ), nil case kind == reflect.Slice || kind == reflect.Array: return makeSliceWriter(typ, ts) case kind == reflect.Struct: return makeStructWriter(typ) - case kind == reflect.Ptr: - return makePtrWriter(typ) + case kind == reflect.Interface: + return writeInterface, nil default: return nil, fmt.Errorf("rlp: type %v is not RLP-serializable", typ) } } -func isByte(typ reflect.Type) bool { - return typ.Kind() == reflect.Uint8 && !typ.Implements(encoderInterface) -} - -func writeRawValue(val reflect.Value, w *encbuf) error { +func writeRawValue(val reflect.Value, w *encBuffer) error { w.str = append(w.str, val.Bytes()...) return nil } -func writeUint(val reflect.Value, w *encbuf) error { - i := val.Uint() - if i == 0 { - w.str = append(w.str, 0x80) - } else if i < 128 { - // fits single byte - w.str = append(w.str, byte(i)) - } else { - // TODO: encode int to w.str directly - s := putint(w.sizebuf[1:], i) - w.sizebuf[0] = 0x80 + byte(s) - w.str = append(w.str, w.sizebuf[:s+1]...) - } +func writeUint(val reflect.Value, w *encBuffer) error { + w.writeUint64(val.Uint()) return nil } -func writeBool(val reflect.Value, w *encbuf) error { - if val.Bool() { - w.str = append(w.str, 0x01) - } else { - w.str = append(w.str, 0x80) - } +func writeBool(val reflect.Value, w *encBuffer) error { + w.writeBool(val.Bool()) return nil } -func writeBigIntPtr(val reflect.Value, w *encbuf) error { +func writeBigIntPtr(val reflect.Value, w *encBuffer) error { ptr := val.Interface().(*big.Int) if ptr == nil { w.str = append(w.str, 0x80) return nil } - return writeBigInt(ptr, w) + if ptr.Sign() == -1 { + return ErrNegativeBigInt + } + w.writeBigInt(ptr) + return nil } -func writeBigIntNoPtr(val reflect.Value, w *encbuf) error { +func writeBigIntNoPtr(val reflect.Value, w *encBuffer) error { i := val.Interface().(big.Int) - return writeBigInt(&i, w) + if i.Sign() == -1 { + return ErrNegativeBigInt + } + w.writeBigInt(&i) + return nil } -func writeBigInt(i *big.Int, w *encbuf) error { - if cmp := i.Cmp(big0); cmp == -1 { - return fmt.Errorf("rlp: cannot encode negative *big.Int") - } else if cmp == 0 { +func writeU256IntPtr(val reflect.Value, w *encBuffer) error { + ptr := val.Interface().(*uint256.Int) + if ptr == nil { w.str = append(w.str, 0x80) - } else { - w.encodeString(i.Bytes()) + return nil } + w.writeUint256(ptr) + return nil +} + +func writeU256IntNoPtr(val reflect.Value, w *encBuffer) error { + i := val.Interface().(uint256.Int) + w.writeUint256(&i) return nil } -func writeBytes(val reflect.Value, w *encbuf) error { - w.encodeString(val.Bytes()) +func writeBytes(val reflect.Value, w *encBuffer) error { + w.writeBytes(val.Bytes()) return nil } -func writeByteArray(val reflect.Value, w *encbuf) error { - if !val.CanAddr() { - // Slice requires the value to be addressable. - // Make it addressable by copying. - copy := reflect.New(val.Type()).Elem() - copy.Set(val) - val = copy +func makeByteArrayWriter(typ reflect.Type) writer { + switch typ.Len() { + case 0: + return writeLengthZeroByteArray + case 1: + return writeLengthOneByteArray + default: + length := typ.Len() + return func(val reflect.Value, w *encBuffer) error { + if !val.CanAddr() { + // Getting the byte slice of val requires it to be addressable. Make it + // addressable by copying. + copy := reflect.New(val.Type()).Elem() + copy.Set(val) + val = copy + } + slice := byteArrayBytes(val, length) + w.encodeStringHeader(len(slice)) + w.str = append(w.str, slice...) + return nil + } } - size := val.Len() - slice := val.Slice(0, size).Bytes() - w.encodeString(slice) +} + +func writeLengthZeroByteArray(val reflect.Value, w *encBuffer) error { + w.str = append(w.str, 0x80) return nil } -func writeString(val reflect.Value, w *encbuf) error { +func writeLengthOneByteArray(val reflect.Value, w *encBuffer) error { + b := byte(val.Index(0).Uint()) + if b <= 0x7f { + w.str = append(w.str, b) + } else { + w.str = append(w.str, 0x81, b) + } + return nil +} + +func writeString(val reflect.Value, w *encBuffer) error { s := val.String() if len(s) == 1 && s[0] <= 0x7f { // fits single byte, no string header @@ -470,27 +284,7 @@ func writeString(val reflect.Value, w *encbuf) error { return nil } -func writeEncoder(val reflect.Value, w *encbuf) error { - return val.Interface().(Encoder).EncodeRLP(w) -} - -// writeEncoderNoPtr handles non-pointer values that implement Encoder -// with a pointer receiver. -func writeEncoderNoPtr(val reflect.Value, w *encbuf) error { - if !val.CanAddr() { - // We can't get the address. It would be possible to make the - // value addressable by creating a shallow copy, but this - // creates other problems so we're not doing it (yet). - // - // package json simply doesn't call MarshalJSON for cases like - // this, but encodes the value as if it didn't implement the - // interface. We don't want to handle it that way. - return fmt.Errorf("rlp: game over: unadressable value of type %v, EncodeRLP is pointer method", val.Type()) - } - return val.Addr().Interface().(Encoder).EncodeRLP(w) -} - -func writeInterface(val reflect.Value, w *encbuf) error { +func writeInterface(val reflect.Value, w *encBuffer) error { if val.IsNil() { // Write empty list. This is consistent with the previous RLP // encoder that we had and should therefore avoid any @@ -499,31 +293,51 @@ func writeInterface(val reflect.Value, w *encbuf) error { return nil } eval := val.Elem() - ti, err := cachedTypeInfo(eval.Type(), tags{}) + writer, err := cachedWriter(eval.Type()) if err != nil { return err } - return ti.writer(eval, w) + return writer(eval, w) } -func makeSliceWriter(typ reflect.Type, ts tags) (writer, error) { - etypeinfo, err := cachedTypeInfo1(typ.Elem(), tags{}) - if err != nil { - return nil, err +func makeSliceWriter(typ reflect.Type, ts rlpstruct.Tags) (writer, error) { + etypeinfo := theTC.infoWhileGenerating(typ.Elem(), rlpstruct.Tags{}) + if etypeinfo.writerErr != nil { + return nil, etypeinfo.writerErr } - writer := func(val reflect.Value, w *encbuf) error { - if !ts.tail { - defer w.listEnd(w.list()) + + var wfn writer + if ts.Tail { + // This is for struct tail slices. + // w.list is not called for them. + wfn = func(val reflect.Value, w *encBuffer) error { + vlen := val.Len() + for i := 0; i < vlen; i++ { + if err := etypeinfo.writer(val.Index(i), w); err != nil { + return err + } + } + return nil } - vlen := val.Len() - for i := 0; i < vlen; i++ { - if err := etypeinfo.writer(val.Index(i), w); err != nil { - return err + } else { + // This is for regular slices and arrays. + wfn = func(val reflect.Value, w *encBuffer) error { + vlen := val.Len() + if vlen == 0 { + w.str = append(w.str, 0xC0) + return nil + } + listOffset := w.list() + for i := 0; i < vlen; i++ { + if err := etypeinfo.writer(val.Index(i), w); err != nil { + return err + } } + w.listEnd(listOffset) + return nil } - return nil } - return writer, nil + return wfn, nil } func makeStructWriter(typ reflect.Type) (writer, error) { @@ -531,56 +345,86 @@ func makeStructWriter(typ reflect.Type) (writer, error) { if err != nil { return nil, err } - writer := func(val reflect.Value, w *encbuf) error { - lh := w.list() - for _, f := range fields { - if err := f.info.writer(val.Field(f.index), w); err != nil { - return err + for _, f := range fields { + if f.info.writerErr != nil { + return nil, structFieldError{typ, f.index, f.info.writerErr} + } + } + + var writer writer + firstOptionalField := firstOptionalField(fields) + if firstOptionalField == len(fields) { + // This is the writer function for structs without any optional fields. + writer = func(val reflect.Value, w *encBuffer) error { + lh := w.list() + for _, f := range fields { + if err := f.info.writer(val.Field(f.index), w); err != nil { + return err + } } + w.listEnd(lh) + return nil + } + } else { + // If there are any "optional" fields, the writer needs to perform additional + // checks to determine the output list length. + writer = func(val reflect.Value, w *encBuffer) error { + lastField := len(fields) - 1 + for ; lastField >= firstOptionalField; lastField-- { + if !val.Field(fields[lastField].index).IsZero() { + break + } + } + lh := w.list() + for i := 0; i <= lastField; i++ { + if err := fields[i].info.writer(val.Field(fields[i].index), w); err != nil { + return err + } + } + w.listEnd(lh) + return nil } - w.listEnd(lh) - return nil } return writer, nil } -func makePtrWriter(typ reflect.Type) (writer, error) { - etypeinfo, err := cachedTypeInfo1(typ.Elem(), tags{}) - if err != nil { - return nil, err +func makePtrWriter(typ reflect.Type, ts rlpstruct.Tags) (writer, error) { + nilEncoding := byte(0xC0) + if typeNilKind(typ.Elem(), ts) == String { + nilEncoding = 0x80 } - // determine nil pointer handler - var nilfunc func(*encbuf) error - kind := typ.Elem().Kind() - switch { - case kind == reflect.Array && isByte(typ.Elem().Elem()): - nilfunc = func(w *encbuf) error { - w.str = append(w.str, 0x80) - return nil - } - case kind == reflect.Struct || kind == reflect.Array: - nilfunc = func(w *encbuf) error { - // encoding the zero value of a struct/array could trigger - // infinite recursion, avoid that. - w.listEnd(w.list()) - return nil - } - default: - zero := reflect.Zero(typ.Elem()) - nilfunc = func(w *encbuf) error { - return etypeinfo.writer(zero, w) + etypeinfo := theTC.infoWhileGenerating(typ.Elem(), rlpstruct.Tags{}) + if etypeinfo.writerErr != nil { + return nil, etypeinfo.writerErr + } + + writer := func(val reflect.Value, w *encBuffer) error { + if ev := val.Elem(); ev.IsValid() { + return etypeinfo.writer(ev, w) } + w.str = append(w.str, nilEncoding) + return nil } + return writer, nil +} - writer := func(val reflect.Value, w *encbuf) error { - if val.IsNil() { - return nilfunc(w) - } else { - return etypeinfo.writer(val.Elem(), w) +func makeEncoderWriter(typ reflect.Type) writer { + if typ.Implements(encoderInterface) { + return func(val reflect.Value, w *encBuffer) error { + return val.Interface().(Encoder).EncodeRLP(w) + } + } + w := func(val reflect.Value, w *encBuffer) error { + if !val.CanAddr() { + // package json simply doesn't call MarshalJSON for this case, but encodes the + // value as if it didn't implement the interface. We don't want to handle it that + // way. + return fmt.Errorf("rlp: unadressable value of type %v, EncodeRLP is pointer method", val.Type()) } + return val.Addr().Interface().(Encoder).EncodeRLP(w) } - return writer, err + return w } // putint writes i to the beginning of b in big endian byte diff --git a/rlp/iterator.go b/rlp/iterator.go new file mode 100644 index 0000000000..6be574572e --- /dev/null +++ b/rlp/iterator.go @@ -0,0 +1,60 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rlp + +type listIterator struct { + data []byte + next []byte + err error +} + +// NewListIterator creates an iterator for the (list) represented by data +// TODO: Consider removing this implementation, as it is no longer used. +func NewListIterator(data RawValue) (*listIterator, error) { + k, t, c, err := readKind(data) + if err != nil { + return nil, err + } + if k != List { + return nil, ErrExpectedList + } + it := &listIterator{ + data: data[t : t+c], + } + return it, nil +} + +// Next forwards the iterator one step, returns true if it was not at end yet +func (it *listIterator) Next() bool { + if len(it.data) == 0 { + return false + } + _, t, c, err := readKind(it.data) + it.next = it.data[:t+c] + it.data = it.data[t+c:] + it.err = err + return true +} + +// Value returns the current value +func (it *listIterator) Value() []byte { + return it.next +} + +func (it *listIterator) Err() error { + return it.err +} diff --git a/rlp/raw.go b/rlp/raw.go index 2b3f328f66..773aa7e614 100644 --- a/rlp/raw.go +++ b/rlp/raw.go @@ -28,12 +28,53 @@ type RawValue []byte var rawValueType = reflect.TypeOf(RawValue{}) +// StringSize returns the encoded size of a string. +func StringSize(s string) uint64 { + switch { + case len(s) == 0: + return 1 + case len(s) == 1: + if s[0] <= 0x7f { + return 1 + } else { + return 2 + } + default: + return uint64(headsize(uint64(len(s))) + len(s)) + } +} + +// BytesSize returns the encoded size of a byte slice. +func BytesSize(b []byte) uint64 { + switch { + case len(b) == 0: + return 1 + case len(b) == 1: + if b[0] <= 0x7f { + return 1 + } else { + return 2 + } + default: + return uint64(headsize(uint64(len(b))) + len(b)) + } +} + // ListSize returns the encoded size of an RLP list with the given // content size. func ListSize(contentSize uint64) uint64 { return uint64(headsize(contentSize)) + contentSize } +// IntSize returns the encoded size of the integer x. Note: The return type of this +// function is 'int' for backwards-compatibility reasons. The result is always positive. +func IntSize(x uint64) int { + if x < 0x80 { + return 1 + } + return 1 + intsize(x) +} + // Split returns the content of first RLP value and any // bytes after the value as subslices of b. func Split(b []byte) (k Kind, content, rest []byte, err error) { @@ -57,6 +98,32 @@ func SplitString(b []byte) (content, rest []byte, err error) { return content, rest, nil } +// SplitUint64 decodes an integer at the beginning of b. +// It also returns the remaining data after the integer in 'rest'. +func SplitUint64(b []byte) (x uint64, rest []byte, err error) { + content, rest, err := SplitString(b) + if err != nil { + return 0, b, err + } + switch { + case len(content) == 0: + return 0, rest, nil + case len(content) == 1: + if content[0] == 0 { + return 0, b, ErrCanonInt + } + return uint64(content[0]), rest, nil + case len(content) > 8: + return 0, b, errUintOverflow + default: + x, err = readSize(content, byte(len(content))) + if err != nil { + return 0, b, ErrCanonInt + } + return x, rest, nil + } +} + // SplitList splits b into the content of a list and any remaining // bytes after the list. func SplitList(b []byte) (content, rest []byte, err error) { @@ -154,3 +221,74 @@ func readSize(b []byte, slen byte) (uint64, error) { } return s, nil } + +// AppendUint64 appends the RLP encoding of i to b, and returns the resulting slice. +func AppendUint64(b []byte, i uint64) []byte { + if i == 0 { + return append(b, 0x80) + } else if i < 128 { + return append(b, byte(i)) + } + switch { + case i < (1 << 8): + return append(b, 0x81, byte(i)) + case i < (1 << 16): + return append(b, 0x82, + byte(i>>8), + byte(i), + ) + case i < (1 << 24): + return append(b, 0x83, + byte(i>>16), + byte(i>>8), + byte(i), + ) + case i < (1 << 32): + return append(b, 0x84, + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) + case i < (1 << 40): + return append(b, 0x85, + byte(i>>32), + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) + + case i < (1 << 48): + return append(b, 0x86, + byte(i>>40), + byte(i>>32), + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) + case i < (1 << 56): + return append(b, 0x87, + byte(i>>48), + byte(i>>40), + byte(i>>32), + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) + + default: + return append(b, 0x88, + byte(i>>56), + byte(i>>48), + byte(i>>40), + byte(i>>32), + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) + } +} diff --git a/rlp/safe.go b/rlp/safe.go new file mode 100644 index 0000000000..3c910337b6 --- /dev/null +++ b/rlp/safe.go @@ -0,0 +1,27 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +//go:build nacl || js || !cgo +// +build nacl js !cgo + +package rlp + +import "reflect" + +// byteArrayBytes returns a slice of the byte array v. +func byteArrayBytes(v reflect.Value, length int) []byte { + return v.Slice(0, length).Bytes() +} diff --git a/rlp/typecache.go b/rlp/typecache.go index 3df799e1ec..c3244050bf 100644 --- a/rlp/typecache.go +++ b/rlp/typecache.go @@ -19,138 +19,222 @@ package rlp import ( "fmt" "reflect" - "strings" "sync" -) + "sync/atomic" -var ( - typeCacheMutex sync.RWMutex - typeCache = make(map[typekey]*typeinfo) + "github.com/tomochain/tomochain/rlp/internal/rlpstruct" ) +// typeinfo is an entry in the type cache. type typeinfo struct { - decoder - writer -} - -// represents struct tags -type tags struct { - // rlp:"nil" controls whether empty input results in a nil pointer. - nilOK bool - // rlp:"tail" controls whether this field swallows additional list - // elements. It can only be set for the last field, which must be - // of slice type. - tail bool - // rlp:"-" ignores fields. - ignored bool + decoder decoder + decoderErr error // error from makeDecoder + writer writer + writerErr error // error from makeWriter } +// typekey is the key of a type in typeCache. It includes the struct tags because +// they might generate a different decoder. type typekey struct { reflect.Type - // the key must include the struct tags because they - // might generate a different decoder. - tags + rlpstruct.Tags } type decoder func(*Stream, reflect.Value) error -type writer func(reflect.Value, *encbuf) error +type writer func(reflect.Value, *encBuffer) error + +var theTC = newTypeCache() + +type typeCache struct { + cur atomic.Value + + // This lock synchronizes writers. + mu sync.Mutex + next map[typekey]*typeinfo +} + +func newTypeCache() *typeCache { + c := new(typeCache) + c.cur.Store(make(map[typekey]*typeinfo)) + return c +} + +func cachedDecoder(typ reflect.Type) (decoder, error) { + info := theTC.info(typ) + return info.decoder, info.decoderErr +} + +func cachedWriter(typ reflect.Type) (writer, error) { + info := theTC.info(typ) + return info.writer, info.writerErr +} + +func (c *typeCache) info(typ reflect.Type) *typeinfo { + key := typekey{Type: typ} + if info := c.cur.Load().(map[typekey]*typeinfo)[key]; info != nil { + return info + } + + // Not in the cache, need to generate info for this type. + return c.generate(typ, rlpstruct.Tags{}) +} + +func (c *typeCache) generate(typ reflect.Type, tags rlpstruct.Tags) *typeinfo { + c.mu.Lock() + defer c.mu.Unlock() + + cur := c.cur.Load().(map[typekey]*typeinfo) + if info := cur[typekey{typ, tags}]; info != nil { + return info + } -func cachedTypeInfo(typ reflect.Type, tags tags) (*typeinfo, error) { - typeCacheMutex.RLock() - info := typeCache[typekey{typ, tags}] - typeCacheMutex.RUnlock() - if info != nil { - return info, nil + // Copy cur to next. + c.next = make(map[typekey]*typeinfo, len(cur)+1) + for k, v := range cur { + c.next[k] = v } - // not in the cache, need to generate info for this type. - typeCacheMutex.Lock() - defer typeCacheMutex.Unlock() - return cachedTypeInfo1(typ, tags) + + // Generate. + info := c.infoWhileGenerating(typ, tags) + + // next -> cur + c.cur.Store(c.next) + c.next = nil + return info } -func cachedTypeInfo1(typ reflect.Type, tags tags) (*typeinfo, error) { +func (c *typeCache) infoWhileGenerating(typ reflect.Type, tags rlpstruct.Tags) *typeinfo { key := typekey{typ, tags} - info := typeCache[key] - if info != nil { - // another goroutine got the write lock first - return info, nil + if info := c.next[key]; info != nil { + return info } - // put a dummmy value into the cache before generating. - // if the generator tries to lookup itself, it will get + // Put a dummy value into the cache before generating. + // If the generator tries to lookup itself, it will get // the dummy value and won't call itself recursively. - typeCache[key] = new(typeinfo) - info, err := genTypeInfo(typ, tags) - if err != nil { - // remove the dummy value if the generator fails - delete(typeCache, key) - return nil, err - } - *typeCache[key] = *info - return typeCache[key], err + info := new(typeinfo) + c.next[key] = info + info.generate(typ, tags) + return info } type field struct { - index int - info *typeinfo + index int + info *typeinfo + optional bool } +// structFields resolves the typeinfo of all public fields in a struct type. func structFields(typ reflect.Type) (fields []field, err error) { + // Convert fields to rlpstruct.Field. + var allStructFields []rlpstruct.Field for i := 0; i < typ.NumField(); i++ { - if f := typ.Field(i); f.PkgPath == "" { // exported - tags, err := parseStructTag(typ, i) - if err != nil { - return nil, err - } - if tags.ignored { - continue - } - info, err := cachedTypeInfo1(f.Type, tags) - if err != nil { - return nil, err - } - fields = append(fields, field{i, info}) + rf := typ.Field(i) + allStructFields = append(allStructFields, rlpstruct.Field{ + Name: rf.Name, + Index: i, + Exported: rf.PkgPath == "", + Tag: string(rf.Tag), + Type: *rtypeToStructType(rf.Type, nil), + }) + } + + // Filter/validate fields. + structFields, structTags, err := rlpstruct.ProcessFields(allStructFields) + if err != nil { + if tagErr, ok := err.(rlpstruct.TagError); ok { + tagErr.StructType = typ.String() + return nil, tagErr } + return nil, err + } + + // Resolve typeinfo. + for i, sf := range structFields { + typ := typ.Field(sf.Index).Type + tags := structTags[i] + info := theTC.infoWhileGenerating(typ, tags) + fields = append(fields, field{sf.Index, info, tags.Optional}) } return fields, nil } -func parseStructTag(typ reflect.Type, fi int) (tags, error) { - f := typ.Field(fi) - var ts tags - for _, t := range strings.Split(f.Tag.Get("rlp"), ",") { - switch t = strings.TrimSpace(t); t { - case "": - case "-": - ts.ignored = true - case "nil": - ts.nilOK = true - case "tail": - ts.tail = true - if fi != typ.NumField()-1 { - return ts, fmt.Errorf(`rlp: invalid struct tag "tail" for %v.%s (must be on last field)`, typ, f.Name) - } - if f.Type.Kind() != reflect.Slice { - return ts, fmt.Errorf(`rlp: invalid struct tag "tail" for %v.%s (field type is not slice)`, typ, f.Name) - } - default: - return ts, fmt.Errorf("rlp: unknown struct tag %q on %v.%s", t, typ, f.Name) +// firstOptionalField returns the index of the first field with "optional" tag. +func firstOptionalField(fields []field) int { + for i, f := range fields { + if f.optional { + return i } } - return ts, nil + return len(fields) } -func genTypeInfo(typ reflect.Type, tags tags) (info *typeinfo, err error) { - info = new(typeinfo) - if info.decoder, err = makeDecoder(typ, tags); err != nil { - return nil, err +type structFieldError struct { + typ reflect.Type + field int + err error +} + +func (e structFieldError) Error() string { + return fmt.Sprintf("%v (struct field %v.%s)", e.err, e.typ, e.typ.Field(e.field).Name) +} + +func (i *typeinfo) generate(typ reflect.Type, tags rlpstruct.Tags) { + i.decoder, i.decoderErr = makeDecoder(typ, tags) + i.writer, i.writerErr = makeWriter(typ, tags) +} + +// rtypeToStructType converts typ to rlpstruct.Type. +func rtypeToStructType(typ reflect.Type, rec map[reflect.Type]*rlpstruct.Type) *rlpstruct.Type { + k := typ.Kind() + if k == reflect.Invalid { + panic("invalid kind") } - if info.writer, err = makeWriter(typ, tags); err != nil { - return nil, err + + if prev := rec[typ]; prev != nil { + return prev // short-circuit for recursive types + } + if rec == nil { + rec = make(map[reflect.Type]*rlpstruct.Type) + } + + t := &rlpstruct.Type{ + Name: typ.String(), + Kind: k, + IsEncoder: typ.Implements(encoderInterface), + IsDecoder: typ.Implements(decoderInterface), + } + rec[typ] = t + if k == reflect.Array || k == reflect.Slice || k == reflect.Ptr { + t.Elem = rtypeToStructType(typ.Elem(), rec) + } + return t +} + +// typeNilKind gives the RLP value kind for nil pointers to 'typ'. +func typeNilKind(typ reflect.Type, tags rlpstruct.Tags) Kind { + styp := rtypeToStructType(typ, nil) + + var nk rlpstruct.NilKind + if tags.NilOK { + nk = tags.NilKind + } else { + nk = styp.DefaultNilValue() + } + switch nk { + case rlpstruct.NilKindString: + return String + case rlpstruct.NilKindList: + return List + default: + panic("invalid nil kind value") } - return info, nil } func isUint(k reflect.Kind) bool { return k >= reflect.Uint && k <= reflect.Uintptr } + +func isByte(typ reflect.Type) bool { + return typ.Kind() == reflect.Uint8 && !typ.Implements(encoderInterface) +} diff --git a/rlp/unsafe.go b/rlp/unsafe.go new file mode 100644 index 0000000000..2152ba35fc --- /dev/null +++ b/rlp/unsafe.go @@ -0,0 +1,35 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +//go:build !nacl && !js && cgo +// +build !nacl,!js,cgo + +package rlp + +import ( + "reflect" + "unsafe" +) + +// byteArrayBytes returns a slice of the byte array v. +func byteArrayBytes(v reflect.Value, length int) []byte { + var s []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&s)) + hdr.Data = v.UnsafeAddr() + hdr.Cap = length + hdr.Len = length + return s +} From 9b5dadb40ac0eb52e8dbe2975864866faec9cd08 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Fri, 16 Jun 2023 16:01:50 +0700 Subject: [PATCH 03/82] Implement RLP encoder code generation tool --- rlp/rlpgen/gen.go | 800 +++++++++++++++++++++++++++ rlp/rlpgen/gen_test.go | 107 ++++ rlp/rlpgen/main.go | 147 +++++ rlp/rlpgen/testdata/bigint.in.txt | 10 + rlp/rlpgen/testdata/bigint.out.txt | 49 ++ rlp/rlpgen/testdata/nil.in.txt | 30 + rlp/rlpgen/testdata/nil.out.txt | 289 ++++++++++ rlp/rlpgen/testdata/optional.in.txt | 17 + rlp/rlpgen/testdata/optional.out.txt | 153 +++++ rlp/rlpgen/testdata/rawvalue.in.txt | 11 + rlp/rlpgen/testdata/rawvalue.out.txt | 64 +++ rlp/rlpgen/testdata/uint256.in.txt | 10 + rlp/rlpgen/testdata/uint256.out.txt | 44 ++ rlp/rlpgen/testdata/uints.in.txt | 10 + rlp/rlpgen/testdata/uints.out.txt | 53 ++ rlp/rlpgen/types.go | 124 +++++ 16 files changed, 1918 insertions(+) create mode 100644 rlp/rlpgen/gen.go create mode 100644 rlp/rlpgen/gen_test.go create mode 100644 rlp/rlpgen/main.go create mode 100644 rlp/rlpgen/testdata/bigint.in.txt create mode 100644 rlp/rlpgen/testdata/bigint.out.txt create mode 100644 rlp/rlpgen/testdata/nil.in.txt create mode 100644 rlp/rlpgen/testdata/nil.out.txt create mode 100644 rlp/rlpgen/testdata/optional.in.txt create mode 100644 rlp/rlpgen/testdata/optional.out.txt create mode 100644 rlp/rlpgen/testdata/rawvalue.in.txt create mode 100644 rlp/rlpgen/testdata/rawvalue.out.txt create mode 100644 rlp/rlpgen/testdata/uint256.in.txt create mode 100644 rlp/rlpgen/testdata/uint256.out.txt create mode 100644 rlp/rlpgen/testdata/uints.in.txt create mode 100644 rlp/rlpgen/testdata/uints.out.txt create mode 100644 rlp/rlpgen/types.go diff --git a/rlp/rlpgen/gen.go b/rlp/rlpgen/gen.go new file mode 100644 index 0000000000..26ccdc574e --- /dev/null +++ b/rlp/rlpgen/gen.go @@ -0,0 +1,800 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package main + +import ( + "bytes" + "fmt" + "go/format" + "go/types" + "sort" + + "github.com/tomochain/tomochain/rlp/internal/rlpstruct" +) + +// buildContext keeps the data needed for make*Op. +type buildContext struct { + topType *types.Named // the type we're creating methods for + + encoderIface *types.Interface + decoderIface *types.Interface + rawValueType *types.Named + + typeToStructCache map[types.Type]*rlpstruct.Type +} + +func newBuildContext(packageRLP *types.Package) *buildContext { + enc := packageRLP.Scope().Lookup("Encoder").Type().Underlying() + dec := packageRLP.Scope().Lookup("Decoder").Type().Underlying() + rawv := packageRLP.Scope().Lookup("RawValue").Type() + return &buildContext{ + typeToStructCache: make(map[types.Type]*rlpstruct.Type), + encoderIface: enc.(*types.Interface), + decoderIface: dec.(*types.Interface), + rawValueType: rawv.(*types.Named), + } +} + +func (bctx *buildContext) isEncoder(typ types.Type) bool { + return types.Implements(typ, bctx.encoderIface) +} + +func (bctx *buildContext) isDecoder(typ types.Type) bool { + return types.Implements(typ, bctx.decoderIface) +} + +// typeToStructType converts typ to rlpstruct.Type. +func (bctx *buildContext) typeToStructType(typ types.Type) *rlpstruct.Type { + if prev := bctx.typeToStructCache[typ]; prev != nil { + return prev // short-circuit for recursive types. + } + + // Resolve named types to their underlying type, but keep the name. + name := types.TypeString(typ, nil) + for { + utype := typ.Underlying() + if utype == typ { + break + } + typ = utype + } + + // Create the type and store it in cache. + t := &rlpstruct.Type{ + Name: name, + Kind: typeReflectKind(typ), + IsEncoder: bctx.isEncoder(typ), + IsDecoder: bctx.isDecoder(typ), + } + bctx.typeToStructCache[typ] = t + + // Assign element type. + switch typ.(type) { + case *types.Array, *types.Slice, *types.Pointer: + etype := typ.(interface{ Elem() types.Type }).Elem() + t.Elem = bctx.typeToStructType(etype) + } + return t +} + +// genContext is passed to the gen* methods of op when generating +// the output code. It tracks packages to be imported by the output +// file and assigns unique names of temporary variables. +type genContext struct { + inPackage *types.Package + imports map[string]struct{} + tempCounter int +} + +func newGenContext(inPackage *types.Package) *genContext { + return &genContext{ + inPackage: inPackage, + imports: make(map[string]struct{}), + } +} + +func (ctx *genContext) temp() string { + v := fmt.Sprintf("_tmp%d", ctx.tempCounter) + ctx.tempCounter++ + return v +} + +func (ctx *genContext) resetTemp() { + ctx.tempCounter = 0 +} + +func (ctx *genContext) addImport(path string) { + if path == ctx.inPackage.Path() { + return // avoid importing the package that we're generating in. + } + // TODO: renaming? + ctx.imports[path] = struct{}{} +} + +// importsList returns all packages that need to be imported. +func (ctx *genContext) importsList() []string { + imp := make([]string, 0, len(ctx.imports)) + for k := range ctx.imports { + imp = append(imp, k) + } + sort.Strings(imp) + return imp +} + +// qualify is the types.Qualifier used for printing types. +func (ctx *genContext) qualify(pkg *types.Package) string { + if pkg.Path() == ctx.inPackage.Path() { + return "" + } + ctx.addImport(pkg.Path()) + // TODO: renaming? + return pkg.Name() +} + +type op interface { + // genWrite creates the encoder. The generated code should write v, + // which is any Go expression, to the rlp.EncoderBuffer 'w'. + genWrite(ctx *genContext, v string) string + + // genDecode creates the decoder. The generated code should read + // a value from the rlp.Stream 'dec' and store it to dst. + genDecode(ctx *genContext) (string, string) +} + +// basicOp handles basic types bool, uint*, string. +type basicOp struct { + typ types.Type + writeMethod string // calle write the value + writeArgType types.Type // parameter type of writeMethod + decMethod string + decResultType types.Type // return type of decMethod + decUseBitSize bool // if true, result bit size is appended to decMethod +} + +func (*buildContext) makeBasicOp(typ *types.Basic) (op, error) { + op := basicOp{typ: typ} + kind := typ.Kind() + switch { + case kind == types.Bool: + op.writeMethod = "WriteBool" + op.writeArgType = types.Typ[types.Bool] + op.decMethod = "Bool" + op.decResultType = types.Typ[types.Bool] + case kind >= types.Uint8 && kind <= types.Uint64: + op.writeMethod = "WriteUint64" + op.writeArgType = types.Typ[types.Uint64] + op.decMethod = "Uint" + op.decResultType = typ + op.decUseBitSize = true + case kind == types.String: + op.writeMethod = "WriteString" + op.writeArgType = types.Typ[types.String] + op.decMethod = "String" + op.decResultType = types.Typ[types.String] + default: + return nil, fmt.Errorf("unhandled basic type: %v", typ) + } + return op, nil +} + +func (*buildContext) makeByteSliceOp(typ *types.Slice) op { + if !isByte(typ.Elem()) { + panic("non-byte slice type in makeByteSliceOp") + } + bslice := types.NewSlice(types.Typ[types.Uint8]) + return basicOp{ + typ: typ, + writeMethod: "WriteBytes", + writeArgType: bslice, + decMethod: "Bytes", + decResultType: bslice, + } +} + +func (bctx *buildContext) makeRawValueOp() op { + bslice := types.NewSlice(types.Typ[types.Uint8]) + return basicOp{ + typ: bctx.rawValueType, + writeMethod: "Write", + writeArgType: bslice, + decMethod: "Raw", + decResultType: bslice, + } +} + +func (op basicOp) writeNeedsConversion() bool { + return !types.AssignableTo(op.typ, op.writeArgType) +} + +func (op basicOp) decodeNeedsConversion() bool { + return !types.AssignableTo(op.decResultType, op.typ) +} + +func (op basicOp) genWrite(ctx *genContext, v string) string { + if op.writeNeedsConversion() { + v = fmt.Sprintf("%s(%s)", op.writeArgType, v) + } + return fmt.Sprintf("w.%s(%s)\n", op.writeMethod, v) +} + +func (op basicOp) genDecode(ctx *genContext) (string, string) { + var ( + resultV = ctx.temp() + result = resultV + method = op.decMethod + ) + if op.decUseBitSize { + // Note: For now, this only works for platform-independent integer + // sizes. makeBasicOp forbids the platform-dependent types. + var sizes types.StdSizes + method = fmt.Sprintf("%s%d", op.decMethod, sizes.Sizeof(op.typ)*8) + } + + // Call the decoder method. + var b bytes.Buffer + fmt.Fprintf(&b, "%s, err := dec.%s()\n", resultV, method) + fmt.Fprintf(&b, "if err != nil { return err }\n") + if op.decodeNeedsConversion() { + conv := ctx.temp() + fmt.Fprintf(&b, "%s := %s(%s)\n", conv, types.TypeString(op.typ, ctx.qualify), resultV) + result = conv + } + return result, b.String() +} + +// byteArrayOp handles [...]byte. +type byteArrayOp struct { + typ types.Type + name types.Type // name != typ for named byte array types (e.g. common.Address) +} + +func (bctx *buildContext) makeByteArrayOp(name *types.Named, typ *types.Array) byteArrayOp { + nt := types.Type(name) + if name == nil { + nt = typ + } + return byteArrayOp{typ, nt} +} + +func (op byteArrayOp) genWrite(ctx *genContext, v string) string { + return fmt.Sprintf("w.WriteBytes(%s[:])\n", v) +} + +func (op byteArrayOp) genDecode(ctx *genContext) (string, string) { + var resultV = ctx.temp() + + var b bytes.Buffer + fmt.Fprintf(&b, "var %s %s\n", resultV, types.TypeString(op.name, ctx.qualify)) + fmt.Fprintf(&b, "if err := dec.ReadBytes(%s[:]); err != nil { return err }\n", resultV) + return resultV, b.String() +} + +// bigIntOp handles big.Int. +// This exists because big.Int has it's own decoder operation on rlp.Stream, +// but the decode method returns *big.Int, so it needs to be dereferenced. +type bigIntOp struct { + pointer bool +} + +func (op bigIntOp) genWrite(ctx *genContext, v string) string { + var b bytes.Buffer + + fmt.Fprintf(&b, "if %s.Sign() == -1 {\n", v) + fmt.Fprintf(&b, " return rlp.ErrNegativeBigInt\n") + fmt.Fprintf(&b, "}\n") + dst := v + if !op.pointer { + dst = "&" + v + } + fmt.Fprintf(&b, "w.WriteBigInt(%s)\n", dst) + + // Wrap with nil check. + if op.pointer { + code := b.String() + b.Reset() + fmt.Fprintf(&b, "if %s == nil {\n", v) + fmt.Fprintf(&b, " w.Write(rlp.EmptyString)") + fmt.Fprintf(&b, "} else {\n") + fmt.Fprint(&b, code) + fmt.Fprintf(&b, "}\n") + } + + return b.String() +} + +func (op bigIntOp) genDecode(ctx *genContext) (string, string) { + var resultV = ctx.temp() + + var b bytes.Buffer + fmt.Fprintf(&b, "%s, err := dec.BigInt()\n", resultV) + fmt.Fprintf(&b, "if err != nil { return err }\n") + + result := resultV + if !op.pointer { + result = "(*" + resultV + ")" + } + return result, b.String() +} + +// uint256Op handles "github.com/holiman/uint256".Int +type uint256Op struct { + pointer bool +} + +func (op uint256Op) genWrite(ctx *genContext, v string) string { + var b bytes.Buffer + + dst := v + if !op.pointer { + dst = "&" + v + } + fmt.Fprintf(&b, "w.WriteUint256(%s)\n", dst) + + // Wrap with nil check. + if op.pointer { + code := b.String() + b.Reset() + fmt.Fprintf(&b, "if %s == nil {\n", v) + fmt.Fprintf(&b, " w.Write(rlp.EmptyString)") + fmt.Fprintf(&b, "} else {\n") + fmt.Fprint(&b, code) + fmt.Fprintf(&b, "}\n") + } + + return b.String() +} + +func (op uint256Op) genDecode(ctx *genContext) (string, string) { + ctx.addImport("github.com/holiman/uint256") + + var b bytes.Buffer + resultV := ctx.temp() + fmt.Fprintf(&b, "var %s uint256.Int\n", resultV) + fmt.Fprintf(&b, "if err := dec.ReadUint256(&%s); err != nil { return err }\n", resultV) + + result := resultV + if op.pointer { + result = "&" + resultV + } + return result, b.String() +} + +// encoderDecoderOp handles rlp.Encoder and rlp.Decoder. +// In order to be used with this, the type must implement both interfaces. +// This restriction may be lifted in the future by creating separate ops for +// encoding and decoding. +type encoderDecoderOp struct { + typ types.Type +} + +func (op encoderDecoderOp) genWrite(ctx *genContext, v string) string { + return fmt.Sprintf("if err := %s.EncodeRLP(w); err != nil { return err }\n", v) +} + +func (op encoderDecoderOp) genDecode(ctx *genContext) (string, string) { + // DecodeRLP must have pointer receiver, and this is verified in makeOp. + etyp := op.typ.(*types.Pointer).Elem() + var resultV = ctx.temp() + + var b bytes.Buffer + fmt.Fprintf(&b, "%s := new(%s)\n", resultV, types.TypeString(etyp, ctx.qualify)) + fmt.Fprintf(&b, "if err := %s.DecodeRLP(dec); err != nil { return err }\n", resultV) + return resultV, b.String() +} + +// ptrOp handles pointer types. +type ptrOp struct { + elemTyp types.Type + elem op + nilOK bool + nilValue rlpstruct.NilKind +} + +func (bctx *buildContext) makePtrOp(elemTyp types.Type, tags rlpstruct.Tags) (op, error) { + elemOp, err := bctx.makeOp(nil, elemTyp, rlpstruct.Tags{}) + if err != nil { + return nil, err + } + op := ptrOp{elemTyp: elemTyp, elem: elemOp} + + // Determine nil value. + if tags.NilOK { + op.nilOK = true + op.nilValue = tags.NilKind + } else { + styp := bctx.typeToStructType(elemTyp) + op.nilValue = styp.DefaultNilValue() + } + return op, nil +} + +func (op ptrOp) genWrite(ctx *genContext, v string) string { + // Note: in writer functions, accesses to v are read-only, i.e. v is any Go + // expression. To make all accesses work through the pointer, we substitute + // v with (*v). This is required for most accesses including `v`, `call(v)`, + // and `v[index]` on slices. + // + // For `v.field` and `v[:]` on arrays, the dereference operation is not required. + var vv string + _, isStruct := op.elem.(structOp) + _, isByteArray := op.elem.(byteArrayOp) + if isStruct || isByteArray { + vv = v + } else { + vv = fmt.Sprintf("(*%s)", v) + } + + var b bytes.Buffer + fmt.Fprintf(&b, "if %s == nil {\n", v) + fmt.Fprintf(&b, " w.Write([]byte{0x%X})\n", op.nilValue) + fmt.Fprintf(&b, "} else {\n") + fmt.Fprintf(&b, " %s", op.elem.genWrite(ctx, vv)) + fmt.Fprintf(&b, "}\n") + return b.String() +} + +func (op ptrOp) genDecode(ctx *genContext) (string, string) { + result, code := op.elem.genDecode(ctx) + if !op.nilOK { + // If nil pointers are not allowed, we can just decode the element. + return "&" + result, code + } + + // nil is allowed, so check the kind and size first. + // If size is zero and kind matches the nilKind of the type, + // the value decodes as a nil pointer. + var ( + resultV = ctx.temp() + kindV = ctx.temp() + sizeV = ctx.temp() + wantKind string + ) + if op.nilValue == rlpstruct.NilKindList { + wantKind = "rlp.List" + } else { + wantKind = "rlp.String" + } + var b bytes.Buffer + fmt.Fprintf(&b, "var %s %s\n", resultV, types.TypeString(types.NewPointer(op.elemTyp), ctx.qualify)) + fmt.Fprintf(&b, "if %s, %s, err := dec.Kind(); err != nil {\n", kindV, sizeV) + fmt.Fprintf(&b, " return err\n") + fmt.Fprintf(&b, "} else if %s != 0 || %s != %s {\n", sizeV, kindV, wantKind) + fmt.Fprint(&b, code) + fmt.Fprintf(&b, " %s = &%s\n", resultV, result) + fmt.Fprintf(&b, "}\n") + return resultV, b.String() +} + +// structOp handles struct types. +type structOp struct { + named *types.Named + typ *types.Struct + fields []*structField + optionalFields []*structField +} + +type structField struct { + name string + typ types.Type + elem op +} + +func (bctx *buildContext) makeStructOp(named *types.Named, typ *types.Struct) (op, error) { + // Convert fields to []rlpstruct.Field. + var allStructFields []rlpstruct.Field + for i := 0; i < typ.NumFields(); i++ { + f := typ.Field(i) + allStructFields = append(allStructFields, rlpstruct.Field{ + Name: f.Name(), + Exported: f.Exported(), + Index: i, + Tag: typ.Tag(i), + Type: *bctx.typeToStructType(f.Type()), + }) + } + + // Filter/validate fields. + fields, tags, err := rlpstruct.ProcessFields(allStructFields) + if err != nil { + return nil, err + } + + // Create field ops. + var op = structOp{named: named, typ: typ} + for i, field := range fields { + // Advanced struct tags are not supported yet. + tag := tags[i] + if err := checkUnsupportedTags(field.Name, tag); err != nil { + return nil, err + } + typ := typ.Field(field.Index).Type() + elem, err := bctx.makeOp(nil, typ, tags[i]) + if err != nil { + return nil, fmt.Errorf("field %s: %v", field.Name, err) + } + f := &structField{name: field.Name, typ: typ, elem: elem} + if tag.Optional { + op.optionalFields = append(op.optionalFields, f) + } else { + op.fields = append(op.fields, f) + } + } + return op, nil +} + +func checkUnsupportedTags(field string, tag rlpstruct.Tags) error { + if tag.Tail { + return fmt.Errorf(`field %s has unsupported struct tag "tail"`, field) + } + return nil +} + +func (op structOp) genWrite(ctx *genContext, v string) string { + var b bytes.Buffer + var listMarker = ctx.temp() + fmt.Fprintf(&b, "%s := w.List()\n", listMarker) + for _, field := range op.fields { + selector := v + "." + field.name + fmt.Fprint(&b, field.elem.genWrite(ctx, selector)) + } + op.writeOptionalFields(&b, ctx, v) + fmt.Fprintf(&b, "w.ListEnd(%s)\n", listMarker) + return b.String() +} + +func (op structOp) writeOptionalFields(b *bytes.Buffer, ctx *genContext, v string) { + if len(op.optionalFields) == 0 { + return + } + // First check zero-ness of all optional fields. + var zeroV = make([]string, len(op.optionalFields)) + for i, field := range op.optionalFields { + selector := v + "." + field.name + zeroV[i] = ctx.temp() + fmt.Fprintf(b, "%s := %s\n", zeroV[i], nonZeroCheck(selector, field.typ, ctx.qualify)) + } + // Now write the fields. + for i, field := range op.optionalFields { + selector := v + "." + field.name + cond := "" + for j := i; j < len(op.optionalFields); j++ { + if j > i { + cond += " || " + } + cond += zeroV[j] + } + fmt.Fprintf(b, "if %s {\n", cond) + fmt.Fprint(b, field.elem.genWrite(ctx, selector)) + fmt.Fprintf(b, "}\n") + } +} + +func (op structOp) genDecode(ctx *genContext) (string, string) { + // Get the string representation of the type. + // Here, named types are handled separately because the output + // would contain a copy of the struct definition otherwise. + var typeName string + if op.named != nil { + typeName = types.TypeString(op.named, ctx.qualify) + } else { + typeName = types.TypeString(op.typ, ctx.qualify) + } + + // Create struct object. + var resultV = ctx.temp() + var b bytes.Buffer + fmt.Fprintf(&b, "var %s %s\n", resultV, typeName) + + // Decode fields. + fmt.Fprintf(&b, "{\n") + fmt.Fprintf(&b, "if _, err := dec.List(); err != nil { return err }\n") + for _, field := range op.fields { + result, code := field.elem.genDecode(ctx) + fmt.Fprintf(&b, "// %s:\n", field.name) + fmt.Fprint(&b, code) + fmt.Fprintf(&b, "%s.%s = %s\n", resultV, field.name, result) + } + op.decodeOptionalFields(&b, ctx, resultV) + fmt.Fprintf(&b, "if err := dec.ListEnd(); err != nil { return err }\n") + fmt.Fprintf(&b, "}\n") + return resultV, b.String() +} + +func (op structOp) decodeOptionalFields(b *bytes.Buffer, ctx *genContext, resultV string) { + var suffix bytes.Buffer + for _, field := range op.optionalFields { + result, code := field.elem.genDecode(ctx) + fmt.Fprintf(b, "// %s:\n", field.name) + fmt.Fprintf(b, "if dec.MoreDataInList() {\n") + fmt.Fprint(b, code) + fmt.Fprintf(b, "%s.%s = %s\n", resultV, field.name, result) + fmt.Fprintf(&suffix, "}\n") + } + suffix.WriteTo(b) +} + +// sliceOp handles slice types. +type sliceOp struct { + typ *types.Slice + elemOp op +} + +func (bctx *buildContext) makeSliceOp(typ *types.Slice) (op, error) { + elemOp, err := bctx.makeOp(nil, typ.Elem(), rlpstruct.Tags{}) + if err != nil { + return nil, err + } + return sliceOp{typ: typ, elemOp: elemOp}, nil +} + +func (op sliceOp) genWrite(ctx *genContext, v string) string { + var ( + listMarker = ctx.temp() // holds return value of w.List() + iterElemV = ctx.temp() // iteration variable + elemCode = op.elemOp.genWrite(ctx, iterElemV) + ) + + var b bytes.Buffer + fmt.Fprintf(&b, "%s := w.List()\n", listMarker) + fmt.Fprintf(&b, "for _, %s := range %s {\n", iterElemV, v) + fmt.Fprint(&b, elemCode) + fmt.Fprintf(&b, "}\n") + fmt.Fprintf(&b, "w.ListEnd(%s)\n", listMarker) + return b.String() +} + +func (op sliceOp) genDecode(ctx *genContext) (string, string) { + var sliceV = ctx.temp() // holds the output slice + elemResult, elemCode := op.elemOp.genDecode(ctx) + + var b bytes.Buffer + fmt.Fprintf(&b, "var %s %s\n", sliceV, types.TypeString(op.typ, ctx.qualify)) + fmt.Fprintf(&b, "if _, err := dec.List(); err != nil { return err }\n") + fmt.Fprintf(&b, "for dec.MoreDataInList() {\n") + fmt.Fprintf(&b, " %s", elemCode) + fmt.Fprintf(&b, " %s = append(%s, %s)\n", sliceV, sliceV, elemResult) + fmt.Fprintf(&b, "}\n") + fmt.Fprintf(&b, "if err := dec.ListEnd(); err != nil { return err }\n") + return sliceV, b.String() +} + +func (bctx *buildContext) makeOp(name *types.Named, typ types.Type, tags rlpstruct.Tags) (op, error) { + switch typ := typ.(type) { + case *types.Named: + if isBigInt(typ) { + return bigIntOp{}, nil + } + if isUint256(typ) { + return uint256Op{}, nil + } + if typ == bctx.rawValueType { + return bctx.makeRawValueOp(), nil + } + if bctx.isDecoder(typ) { + return nil, fmt.Errorf("type %v implements rlp.Decoder with non-pointer receiver", typ) + } + // TODO: same check for encoder? + return bctx.makeOp(typ, typ.Underlying(), tags) + case *types.Pointer: + if isBigInt(typ.Elem()) { + return bigIntOp{pointer: true}, nil + } + if isUint256(typ.Elem()) { + return uint256Op{pointer: true}, nil + } + // Encoder/Decoder interfaces. + if bctx.isEncoder(typ) { + if bctx.isDecoder(typ) { + return encoderDecoderOp{typ}, nil + } + return nil, fmt.Errorf("type %v implements rlp.Encoder but not rlp.Decoder", typ) + } + if bctx.isDecoder(typ) { + return nil, fmt.Errorf("type %v implements rlp.Decoder but not rlp.Encoder", typ) + } + // Default pointer handling. + return bctx.makePtrOp(typ.Elem(), tags) + case *types.Basic: + return bctx.makeBasicOp(typ) + case *types.Struct: + return bctx.makeStructOp(name, typ) + case *types.Slice: + etyp := typ.Elem() + if isByte(etyp) && !bctx.isEncoder(etyp) { + return bctx.makeByteSliceOp(typ), nil + } + return bctx.makeSliceOp(typ) + case *types.Array: + etyp := typ.Elem() + if isByte(etyp) && !bctx.isEncoder(etyp) { + return bctx.makeByteArrayOp(name, typ), nil + } + return nil, fmt.Errorf("unhandled array type: %v", typ) + default: + return nil, fmt.Errorf("unhandled type: %v", typ) + } +} + +// generateDecoder generates the DecodeRLP method on 'typ'. +func generateDecoder(ctx *genContext, typ string, op op) []byte { + ctx.resetTemp() + ctx.addImport(pathOfPackageRLP) + + result, code := op.genDecode(ctx) + var b bytes.Buffer + fmt.Fprintf(&b, "func (obj *%s) DecodeRLP(dec *rlp.Stream) error {\n", typ) + fmt.Fprint(&b, code) + fmt.Fprintf(&b, " *obj = %s\n", result) + fmt.Fprintf(&b, " return nil\n") + fmt.Fprintf(&b, "}\n") + return b.Bytes() +} + +// generateEncoder generates the EncodeRLP method on 'typ'. +func generateEncoder(ctx *genContext, typ string, op op) []byte { + ctx.resetTemp() + ctx.addImport("io") + ctx.addImport(pathOfPackageRLP) + + var b bytes.Buffer + fmt.Fprintf(&b, "func (obj *%s) EncodeRLP(_w io.Writer) error {\n", typ) + fmt.Fprintf(&b, " w := rlp.NewEncoderBuffer(_w)\n") + fmt.Fprint(&b, op.genWrite(ctx, "obj")) + fmt.Fprintf(&b, " return w.Flush()\n") + fmt.Fprintf(&b, "}\n") + return b.Bytes() +} + +func (bctx *buildContext) generate(typ *types.Named, encoder, decoder bool) ([]byte, error) { + bctx.topType = typ + + pkg := typ.Obj().Pkg() + op, err := bctx.makeOp(nil, typ, rlpstruct.Tags{}) + if err != nil { + return nil, err + } + + var ( + ctx = newGenContext(pkg) + encSource []byte + decSource []byte + ) + if encoder { + encSource = generateEncoder(ctx, typ.Obj().Name(), op) + } + if decoder { + decSource = generateDecoder(ctx, typ.Obj().Name(), op) + } + + var b bytes.Buffer + fmt.Fprintf(&b, "package %s\n\n", pkg.Name()) + for _, imp := range ctx.importsList() { + fmt.Fprintf(&b, "import %q\n", imp) + } + if encoder { + fmt.Fprintln(&b) + b.Write(encSource) + } + if decoder { + fmt.Fprintln(&b) + b.Write(decSource) + } + + source := b.Bytes() + // fmt.Println(string(source)) + return format.Source(source) +} diff --git a/rlp/rlpgen/gen_test.go b/rlp/rlpgen/gen_test.go new file mode 100644 index 0000000000..3b4f5df287 --- /dev/null +++ b/rlp/rlpgen/gen_test.go @@ -0,0 +1,107 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package main + +import ( + "bytes" + "fmt" + "go/ast" + "go/importer" + "go/parser" + "go/token" + "go/types" + "os" + "path/filepath" + "testing" +) + +// Package RLP is loaded only once and reused for all tests. +var ( + testFset = token.NewFileSet() + testImporter = importer.ForCompiler(testFset, "source", nil).(types.ImporterFrom) + testPackageRLP *types.Package +) + +func init() { + cwd, err := os.Getwd() + if err != nil { + panic(err) + } + testPackageRLP, err = testImporter.ImportFrom(pathOfPackageRLP, cwd, 0) + if err != nil { + panic(fmt.Errorf("can't load package RLP: %v", err)) + } +} + +var tests = []string{"uints", "nil", "rawvalue", "optional", "bigint", "uint256"} + +func TestOutput(t *testing.T) { + for _, test := range tests { + test := test + t.Run(test, func(t *testing.T) { + inputFile := filepath.Join("testdata", test+".in.txt") + outputFile := filepath.Join("testdata", test+".out.txt") + bctx, typ, err := loadTestSource(inputFile, "Test") + if err != nil { + t.Fatal("error loading test source:", err) + } + output, err := bctx.generate(typ, true, true) + if err != nil { + t.Fatal("error in generate:", err) + } + + // Set this environment variable to regenerate the test outputs. + if os.Getenv("WRITE_TEST_FILES") != "" { + os.WriteFile(outputFile, output, 0644) + } + + // Check if output matches. + wantOutput, err := os.ReadFile(outputFile) + if err != nil { + t.Fatal("error loading expected test output:", err) + } + if !bytes.Equal(output, wantOutput) { + t.Fatalf("output mismatch, want: %v got %v", string(wantOutput), string(output)) + } + }) + } +} + +func loadTestSource(file string, typeName string) (*buildContext, *types.Named, error) { + // Load the test input. + content, err := os.ReadFile(file) + if err != nil { + return nil, nil, err + } + f, err := parser.ParseFile(testFset, file, content, 0) + if err != nil { + return nil, nil, err + } + conf := types.Config{Importer: testImporter} + pkg, err := conf.Check("test", testFset, []*ast.File{f}, nil) + if err != nil { + return nil, nil, err + } + + // Find the test struct. + bctx := newBuildContext(testPackageRLP) + typ, err := lookupStructType(pkg.Scope(), typeName) + if err != nil { + return nil, nil, fmt.Errorf("can't find type %s: %v", typeName, err) + } + return bctx, typ, nil +} diff --git a/rlp/rlpgen/main.go b/rlp/rlpgen/main.go new file mode 100644 index 0000000000..87aebbc47a --- /dev/null +++ b/rlp/rlpgen/main.go @@ -0,0 +1,147 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package main + +import ( + "bytes" + "errors" + "flag" + "fmt" + "go/types" + "os" + + "golang.org/x/tools/go/packages" +) + +const pathOfPackageRLP = "github.com/tomochain/tomochain/rlp" + +func main() { + var ( + pkgdir = flag.String("dir", ".", "input package") + output = flag.String("out", "-", "output file (default is stdout)") + genEncoder = flag.Bool("encoder", true, "generate EncodeRLP?") + genDecoder = flag.Bool("decoder", false, "generate DecodeRLP?") + typename = flag.String("type", "", "type to generate methods for") + ) + flag.Parse() + + cfg := Config{ + Dir: *pkgdir, + Type: *typename, + GenerateEncoder: *genEncoder, + GenerateDecoder: *genDecoder, + } + code, err := cfg.process() + if err != nil { + fatal(err) + } + if *output == "-" { + os.Stdout.Write(code) + } else if err := os.WriteFile(*output, code, 0600); err != nil { + fatal(err) + } +} + +func fatal(args ...interface{}) { + fmt.Fprintln(os.Stderr, args...) + os.Exit(1) +} + +type Config struct { + Dir string // input package directory + Type string + + GenerateEncoder bool + GenerateDecoder bool +} + +// process generates the Go code. +func (cfg *Config) process() (code []byte, err error) { + // Load packages. + pcfg := &packages.Config{ + Mode: packages.NeedName | packages.NeedTypes | packages.NeedImports | packages.NeedDeps, + Dir: cfg.Dir, + BuildFlags: []string{"-tags", "norlpgen"}, + } + ps, err := packages.Load(pcfg, pathOfPackageRLP, ".") + if err != nil { + return nil, err + } + if len(ps) == 0 { + return nil, fmt.Errorf("no Go package found in %s", cfg.Dir) + } + packages.PrintErrors(ps) + + // Find the packages that were loaded. + var ( + pkg *types.Package + packageRLP *types.Package + ) + for _, p := range ps { + if len(p.Errors) > 0 { + return nil, fmt.Errorf("package %s has errors", p.PkgPath) + } + if p.PkgPath == pathOfPackageRLP { + packageRLP = p.Types + } else { + pkg = p.Types + } + } + bctx := newBuildContext(packageRLP) + + // Find the type and generate. + typ, err := lookupStructType(pkg.Scope(), cfg.Type) + if err != nil { + return nil, fmt.Errorf("can't find %s in %s: %v", cfg.Type, pkg, err) + } + code, err = bctx.generate(typ, cfg.GenerateEncoder, cfg.GenerateDecoder) + if err != nil { + return nil, err + } + + // Add build comments. + // This is done here to avoid processing these lines with gofmt. + var header bytes.Buffer + fmt.Fprint(&header, "// Code generated by rlpgen. DO NOT EDIT.\n\n") + fmt.Fprint(&header, "//go:build !norlpgen\n") + fmt.Fprint(&header, "// +build !norlpgen\n\n") + return append(header.Bytes(), code...), nil +} + +func lookupStructType(scope *types.Scope, name string) (*types.Named, error) { + typ, err := lookupType(scope, name) + if err != nil { + return nil, err + } + _, ok := typ.Underlying().(*types.Struct) + if !ok { + return nil, errors.New("not a struct type") + } + return typ, nil +} + +func lookupType(scope *types.Scope, name string) (*types.Named, error) { + obj := scope.Lookup(name) + if obj == nil { + return nil, errors.New("no such identifier") + } + typ, ok := obj.(*types.TypeName) + if !ok { + return nil, errors.New("not a type") + } + return typ.Type().(*types.Named), nil +} diff --git a/rlp/rlpgen/testdata/bigint.in.txt b/rlp/rlpgen/testdata/bigint.in.txt new file mode 100644 index 0000000000..d23d84a287 --- /dev/null +++ b/rlp/rlpgen/testdata/bigint.in.txt @@ -0,0 +1,10 @@ +// -*- mode: go -*- + +package test + +import "math/big" + +type Test struct { + Int *big.Int + IntNoPtr big.Int +} diff --git a/rlp/rlpgen/testdata/bigint.out.txt b/rlp/rlpgen/testdata/bigint.out.txt new file mode 100644 index 0000000000..6dc7bea3bf --- /dev/null +++ b/rlp/rlpgen/testdata/bigint.out.txt @@ -0,0 +1,49 @@ +package test + +import "github.com/tomochain/tomochain/rlp" +import "io" + +func (obj *Test) EncodeRLP(_w io.Writer) error { + w := rlp.NewEncoderBuffer(_w) + _tmp0 := w.List() + if obj.Int == nil { + w.Write(rlp.EmptyString) + } else { + if obj.Int.Sign() == -1 { + return rlp.ErrNegativeBigInt + } + w.WriteBigInt(obj.Int) + } + if obj.IntNoPtr.Sign() == -1 { + return rlp.ErrNegativeBigInt + } + w.WriteBigInt(&obj.IntNoPtr) + w.ListEnd(_tmp0) + return w.Flush() +} + +func (obj *Test) DecodeRLP(dec *rlp.Stream) error { + var _tmp0 Test + { + if _, err := dec.List(); err != nil { + return err + } + // Int: + _tmp1, err := dec.BigInt() + if err != nil { + return err + } + _tmp0.Int = _tmp1 + // IntNoPtr: + _tmp2, err := dec.BigInt() + if err != nil { + return err + } + _tmp0.IntNoPtr = (*_tmp2) + if err := dec.ListEnd(); err != nil { + return err + } + } + *obj = _tmp0 + return nil +} diff --git a/rlp/rlpgen/testdata/nil.in.txt b/rlp/rlpgen/testdata/nil.in.txt new file mode 100644 index 0000000000..a28ff34487 --- /dev/null +++ b/rlp/rlpgen/testdata/nil.in.txt @@ -0,0 +1,30 @@ +// -*- mode: go -*- + +package test + +type Aux struct{ + A uint32 +} + +type Test struct{ + Uint8 *byte `rlp:"nil"` + Uint8List *byte `rlp:"nilList"` + + Uint32 *uint32 `rlp:"nil"` + Uint32List *uint32 `rlp:"nilList"` + + Uint64 *uint64 `rlp:"nil"` + Uint64List *uint64 `rlp:"nilList"` + + String *string `rlp:"nil"` + StringList *string `rlp:"nilList"` + + ByteArray *[3]byte `rlp:"nil"` + ByteArrayList *[3]byte `rlp:"nilList"` + + ByteSlice *[]byte `rlp:"nil"` + ByteSliceList *[]byte `rlp:"nilList"` + + Struct *Aux `rlp:"nil"` + StructString *Aux `rlp:"nilString"` +} diff --git a/rlp/rlpgen/testdata/nil.out.txt b/rlp/rlpgen/testdata/nil.out.txt new file mode 100644 index 0000000000..b3bdd0b86f --- /dev/null +++ b/rlp/rlpgen/testdata/nil.out.txt @@ -0,0 +1,289 @@ +package test + +import "github.com/tomochain/tomochain/rlp" +import "io" + +func (obj *Test) EncodeRLP(_w io.Writer) error { + w := rlp.NewEncoderBuffer(_w) + _tmp0 := w.List() + if obj.Uint8 == nil { + w.Write([]byte{0x80}) + } else { + w.WriteUint64(uint64((*obj.Uint8))) + } + if obj.Uint8List == nil { + w.Write([]byte{0xC0}) + } else { + w.WriteUint64(uint64((*obj.Uint8List))) + } + if obj.Uint32 == nil { + w.Write([]byte{0x80}) + } else { + w.WriteUint64(uint64((*obj.Uint32))) + } + if obj.Uint32List == nil { + w.Write([]byte{0xC0}) + } else { + w.WriteUint64(uint64((*obj.Uint32List))) + } + if obj.Uint64 == nil { + w.Write([]byte{0x80}) + } else { + w.WriteUint64((*obj.Uint64)) + } + if obj.Uint64List == nil { + w.Write([]byte{0xC0}) + } else { + w.WriteUint64((*obj.Uint64List)) + } + if obj.String == nil { + w.Write([]byte{0x80}) + } else { + w.WriteString((*obj.String)) + } + if obj.StringList == nil { + w.Write([]byte{0xC0}) + } else { + w.WriteString((*obj.StringList)) + } + if obj.ByteArray == nil { + w.Write([]byte{0x80}) + } else { + w.WriteBytes(obj.ByteArray[:]) + } + if obj.ByteArrayList == nil { + w.Write([]byte{0xC0}) + } else { + w.WriteBytes(obj.ByteArrayList[:]) + } + if obj.ByteSlice == nil { + w.Write([]byte{0x80}) + } else { + w.WriteBytes((*obj.ByteSlice)) + } + if obj.ByteSliceList == nil { + w.Write([]byte{0xC0}) + } else { + w.WriteBytes((*obj.ByteSliceList)) + } + if obj.Struct == nil { + w.Write([]byte{0xC0}) + } else { + _tmp1 := w.List() + w.WriteUint64(uint64(obj.Struct.A)) + w.ListEnd(_tmp1) + } + if obj.StructString == nil { + w.Write([]byte{0x80}) + } else { + _tmp2 := w.List() + w.WriteUint64(uint64(obj.StructString.A)) + w.ListEnd(_tmp2) + } + w.ListEnd(_tmp0) + return w.Flush() +} + +func (obj *Test) DecodeRLP(dec *rlp.Stream) error { + var _tmp0 Test + { + if _, err := dec.List(); err != nil { + return err + } + // Uint8: + var _tmp2 *byte + if _tmp3, _tmp4, err := dec.Kind(); err != nil { + return err + } else if _tmp4 != 0 || _tmp3 != rlp.String { + _tmp1, err := dec.Uint8() + if err != nil { + return err + } + _tmp2 = &_tmp1 + } + _tmp0.Uint8 = _tmp2 + // Uint8List: + var _tmp6 *byte + if _tmp7, _tmp8, err := dec.Kind(); err != nil { + return err + } else if _tmp8 != 0 || _tmp7 != rlp.List { + _tmp5, err := dec.Uint8() + if err != nil { + return err + } + _tmp6 = &_tmp5 + } + _tmp0.Uint8List = _tmp6 + // Uint32: + var _tmp10 *uint32 + if _tmp11, _tmp12, err := dec.Kind(); err != nil { + return err + } else if _tmp12 != 0 || _tmp11 != rlp.String { + _tmp9, err := dec.Uint32() + if err != nil { + return err + } + _tmp10 = &_tmp9 + } + _tmp0.Uint32 = _tmp10 + // Uint32List: + var _tmp14 *uint32 + if _tmp15, _tmp16, err := dec.Kind(); err != nil { + return err + } else if _tmp16 != 0 || _tmp15 != rlp.List { + _tmp13, err := dec.Uint32() + if err != nil { + return err + } + _tmp14 = &_tmp13 + } + _tmp0.Uint32List = _tmp14 + // Uint64: + var _tmp18 *uint64 + if _tmp19, _tmp20, err := dec.Kind(); err != nil { + return err + } else if _tmp20 != 0 || _tmp19 != rlp.String { + _tmp17, err := dec.Uint64() + if err != nil { + return err + } + _tmp18 = &_tmp17 + } + _tmp0.Uint64 = _tmp18 + // Uint64List: + var _tmp22 *uint64 + if _tmp23, _tmp24, err := dec.Kind(); err != nil { + return err + } else if _tmp24 != 0 || _tmp23 != rlp.List { + _tmp21, err := dec.Uint64() + if err != nil { + return err + } + _tmp22 = &_tmp21 + } + _tmp0.Uint64List = _tmp22 + // String: + var _tmp26 *string + if _tmp27, _tmp28, err := dec.Kind(); err != nil { + return err + } else if _tmp28 != 0 || _tmp27 != rlp.String { + _tmp25, err := dec.String() + if err != nil { + return err + } + _tmp26 = &_tmp25 + } + _tmp0.String = _tmp26 + // StringList: + var _tmp30 *string + if _tmp31, _tmp32, err := dec.Kind(); err != nil { + return err + } else if _tmp32 != 0 || _tmp31 != rlp.List { + _tmp29, err := dec.String() + if err != nil { + return err + } + _tmp30 = &_tmp29 + } + _tmp0.StringList = _tmp30 + // ByteArray: + var _tmp34 *[3]byte + if _tmp35, _tmp36, err := dec.Kind(); err != nil { + return err + } else if _tmp36 != 0 || _tmp35 != rlp.String { + var _tmp33 [3]byte + if err := dec.ReadBytes(_tmp33[:]); err != nil { + return err + } + _tmp34 = &_tmp33 + } + _tmp0.ByteArray = _tmp34 + // ByteArrayList: + var _tmp38 *[3]byte + if _tmp39, _tmp40, err := dec.Kind(); err != nil { + return err + } else if _tmp40 != 0 || _tmp39 != rlp.List { + var _tmp37 [3]byte + if err := dec.ReadBytes(_tmp37[:]); err != nil { + return err + } + _tmp38 = &_tmp37 + } + _tmp0.ByteArrayList = _tmp38 + // ByteSlice: + var _tmp42 *[]byte + if _tmp43, _tmp44, err := dec.Kind(); err != nil { + return err + } else if _tmp44 != 0 || _tmp43 != rlp.String { + _tmp41, err := dec.Bytes() + if err != nil { + return err + } + _tmp42 = &_tmp41 + } + _tmp0.ByteSlice = _tmp42 + // ByteSliceList: + var _tmp46 *[]byte + if _tmp47, _tmp48, err := dec.Kind(); err != nil { + return err + } else if _tmp48 != 0 || _tmp47 != rlp.List { + _tmp45, err := dec.Bytes() + if err != nil { + return err + } + _tmp46 = &_tmp45 + } + _tmp0.ByteSliceList = _tmp46 + // Struct: + var _tmp51 *Aux + if _tmp52, _tmp53, err := dec.Kind(); err != nil { + return err + } else if _tmp53 != 0 || _tmp52 != rlp.List { + var _tmp49 Aux + { + if _, err := dec.List(); err != nil { + return err + } + // A: + _tmp50, err := dec.Uint32() + if err != nil { + return err + } + _tmp49.A = _tmp50 + if err := dec.ListEnd(); err != nil { + return err + } + } + _tmp51 = &_tmp49 + } + _tmp0.Struct = _tmp51 + // StructString: + var _tmp56 *Aux + if _tmp57, _tmp58, err := dec.Kind(); err != nil { + return err + } else if _tmp58 != 0 || _tmp57 != rlp.String { + var _tmp54 Aux + { + if _, err := dec.List(); err != nil { + return err + } + // A: + _tmp55, err := dec.Uint32() + if err != nil { + return err + } + _tmp54.A = _tmp55 + if err := dec.ListEnd(); err != nil { + return err + } + } + _tmp56 = &_tmp54 + } + _tmp0.StructString = _tmp56 + if err := dec.ListEnd(); err != nil { + return err + } + } + *obj = _tmp0 + return nil +} diff --git a/rlp/rlpgen/testdata/optional.in.txt b/rlp/rlpgen/testdata/optional.in.txt new file mode 100644 index 0000000000..f1ac9f7899 --- /dev/null +++ b/rlp/rlpgen/testdata/optional.in.txt @@ -0,0 +1,17 @@ +// -*- mode: go -*- + +package test + +type Aux struct { + A uint64 +} + +type Test struct { + Uint64 uint64 `rlp:"optional"` + Pointer *uint64 `rlp:"optional"` + String string `rlp:"optional"` + Slice []uint64 `rlp:"optional"` + Array [3]byte `rlp:"optional"` + NamedStruct Aux `rlp:"optional"` + AnonStruct struct{ A string } `rlp:"optional"` +} diff --git a/rlp/rlpgen/testdata/optional.out.txt b/rlp/rlpgen/testdata/optional.out.txt new file mode 100644 index 0000000000..fb9b95d44d --- /dev/null +++ b/rlp/rlpgen/testdata/optional.out.txt @@ -0,0 +1,153 @@ +package test + +import "github.com/tomochain/tomochain/rlp" +import "io" + +func (obj *Test) EncodeRLP(_w io.Writer) error { + w := rlp.NewEncoderBuffer(_w) + _tmp0 := w.List() + _tmp1 := obj.Uint64 != 0 + _tmp2 := obj.Pointer != nil + _tmp3 := obj.String != "" + _tmp4 := len(obj.Slice) > 0 + _tmp5 := obj.Array != ([3]byte{}) + _tmp6 := obj.NamedStruct != (Aux{}) + _tmp7 := obj.AnonStruct != (struct{ A string }{}) + if _tmp1 || _tmp2 || _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 { + w.WriteUint64(obj.Uint64) + } + if _tmp2 || _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 { + if obj.Pointer == nil { + w.Write([]byte{0x80}) + } else { + w.WriteUint64((*obj.Pointer)) + } + } + if _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 { + w.WriteString(obj.String) + } + if _tmp4 || _tmp5 || _tmp6 || _tmp7 { + _tmp8 := w.List() + for _, _tmp9 := range obj.Slice { + w.WriteUint64(_tmp9) + } + w.ListEnd(_tmp8) + } + if _tmp5 || _tmp6 || _tmp7 { + w.WriteBytes(obj.Array[:]) + } + if _tmp6 || _tmp7 { + _tmp10 := w.List() + w.WriteUint64(obj.NamedStruct.A) + w.ListEnd(_tmp10) + } + if _tmp7 { + _tmp11 := w.List() + w.WriteString(obj.AnonStruct.A) + w.ListEnd(_tmp11) + } + w.ListEnd(_tmp0) + return w.Flush() +} + +func (obj *Test) DecodeRLP(dec *rlp.Stream) error { + var _tmp0 Test + { + if _, err := dec.List(); err != nil { + return err + } + // Uint64: + if dec.MoreDataInList() { + _tmp1, err := dec.Uint64() + if err != nil { + return err + } + _tmp0.Uint64 = _tmp1 + // Pointer: + if dec.MoreDataInList() { + _tmp2, err := dec.Uint64() + if err != nil { + return err + } + _tmp0.Pointer = &_tmp2 + // String: + if dec.MoreDataInList() { + _tmp3, err := dec.String() + if err != nil { + return err + } + _tmp0.String = _tmp3 + // Slice: + if dec.MoreDataInList() { + var _tmp4 []uint64 + if _, err := dec.List(); err != nil { + return err + } + for dec.MoreDataInList() { + _tmp5, err := dec.Uint64() + if err != nil { + return err + } + _tmp4 = append(_tmp4, _tmp5) + } + if err := dec.ListEnd(); err != nil { + return err + } + _tmp0.Slice = _tmp4 + // Array: + if dec.MoreDataInList() { + var _tmp6 [3]byte + if err := dec.ReadBytes(_tmp6[:]); err != nil { + return err + } + _tmp0.Array = _tmp6 + // NamedStruct: + if dec.MoreDataInList() { + var _tmp7 Aux + { + if _, err := dec.List(); err != nil { + return err + } + // A: + _tmp8, err := dec.Uint64() + if err != nil { + return err + } + _tmp7.A = _tmp8 + if err := dec.ListEnd(); err != nil { + return err + } + } + _tmp0.NamedStruct = _tmp7 + // AnonStruct: + if dec.MoreDataInList() { + var _tmp9 struct{ A string } + { + if _, err := dec.List(); err != nil { + return err + } + // A: + _tmp10, err := dec.String() + if err != nil { + return err + } + _tmp9.A = _tmp10 + if err := dec.ListEnd(); err != nil { + return err + } + } + _tmp0.AnonStruct = _tmp9 + } + } + } + } + } + } + } + if err := dec.ListEnd(); err != nil { + return err + } + } + *obj = _tmp0 + return nil +} diff --git a/rlp/rlpgen/testdata/rawvalue.in.txt b/rlp/rlpgen/testdata/rawvalue.in.txt new file mode 100644 index 0000000000..6c17849954 --- /dev/null +++ b/rlp/rlpgen/testdata/rawvalue.in.txt @@ -0,0 +1,11 @@ +// -*- mode: go -*- + +package test + +import "github.com/tomochain/tomochain/rlp" + +type Test struct { + RawValue rlp.RawValue + PointerToRawValue *rlp.RawValue + SliceOfRawValue []rlp.RawValue +} diff --git a/rlp/rlpgen/testdata/rawvalue.out.txt b/rlp/rlpgen/testdata/rawvalue.out.txt new file mode 100644 index 0000000000..4b6eb385d6 --- /dev/null +++ b/rlp/rlpgen/testdata/rawvalue.out.txt @@ -0,0 +1,64 @@ +package test + +import "github.com/tomochain/tomochain/rlp" +import "io" + +func (obj *Test) EncodeRLP(_w io.Writer) error { + w := rlp.NewEncoderBuffer(_w) + _tmp0 := w.List() + w.Write(obj.RawValue) + if obj.PointerToRawValue == nil { + w.Write([]byte{0x80}) + } else { + w.Write((*obj.PointerToRawValue)) + } + _tmp1 := w.List() + for _, _tmp2 := range obj.SliceOfRawValue { + w.Write(_tmp2) + } + w.ListEnd(_tmp1) + w.ListEnd(_tmp0) + return w.Flush() +} + +func (obj *Test) DecodeRLP(dec *rlp.Stream) error { + var _tmp0 Test + { + if _, err := dec.List(); err != nil { + return err + } + // RawValue: + _tmp1, err := dec.Raw() + if err != nil { + return err + } + _tmp0.RawValue = _tmp1 + // PointerToRawValue: + _tmp2, err := dec.Raw() + if err != nil { + return err + } + _tmp0.PointerToRawValue = &_tmp2 + // SliceOfRawValue: + var _tmp3 []rlp.RawValue + if _, err := dec.List(); err != nil { + return err + } + for dec.MoreDataInList() { + _tmp4, err := dec.Raw() + if err != nil { + return err + } + _tmp3 = append(_tmp3, _tmp4) + } + if err := dec.ListEnd(); err != nil { + return err + } + _tmp0.SliceOfRawValue = _tmp3 + if err := dec.ListEnd(); err != nil { + return err + } + } + *obj = _tmp0 + return nil +} diff --git a/rlp/rlpgen/testdata/uint256.in.txt b/rlp/rlpgen/testdata/uint256.in.txt new file mode 100644 index 0000000000..ed16e0a788 --- /dev/null +++ b/rlp/rlpgen/testdata/uint256.in.txt @@ -0,0 +1,10 @@ +// -*- mode: go -*- + +package test + +import "github.com/holiman/uint256" + +type Test struct { + Int *uint256.Int + IntNoPtr uint256.Int +} diff --git a/rlp/rlpgen/testdata/uint256.out.txt b/rlp/rlpgen/testdata/uint256.out.txt new file mode 100644 index 0000000000..5d99ca2e6d --- /dev/null +++ b/rlp/rlpgen/testdata/uint256.out.txt @@ -0,0 +1,44 @@ +package test + +import "github.com/holiman/uint256" +import "github.com/tomochain/tomochain/rlp" +import "io" + +func (obj *Test) EncodeRLP(_w io.Writer) error { + w := rlp.NewEncoderBuffer(_w) + _tmp0 := w.List() + if obj.Int == nil { + w.Write(rlp.EmptyString) + } else { + w.WriteUint256(obj.Int) + } + w.WriteUint256(&obj.IntNoPtr) + w.ListEnd(_tmp0) + return w.Flush() +} + +func (obj *Test) DecodeRLP(dec *rlp.Stream) error { + var _tmp0 Test + { + if _, err := dec.List(); err != nil { + return err + } + // Int: + var _tmp1 uint256.Int + if err := dec.ReadUint256(&_tmp1); err != nil { + return err + } + _tmp0.Int = &_tmp1 + // IntNoPtr: + var _tmp2 uint256.Int + if err := dec.ReadUint256(&_tmp2); err != nil { + return err + } + _tmp0.IntNoPtr = _tmp2 + if err := dec.ListEnd(); err != nil { + return err + } + } + *obj = _tmp0 + return nil +} diff --git a/rlp/rlpgen/testdata/uints.in.txt b/rlp/rlpgen/testdata/uints.in.txt new file mode 100644 index 0000000000..8095da997d --- /dev/null +++ b/rlp/rlpgen/testdata/uints.in.txt @@ -0,0 +1,10 @@ +// -*- mode: go -*- + +package test + +type Test struct{ + A uint8 + B uint16 + C uint32 + D uint64 +} diff --git a/rlp/rlpgen/testdata/uints.out.txt b/rlp/rlpgen/testdata/uints.out.txt new file mode 100644 index 0000000000..17896dd305 --- /dev/null +++ b/rlp/rlpgen/testdata/uints.out.txt @@ -0,0 +1,53 @@ +package test + +import "github.com/tomochain/tomochain/rlp" +import "io" + +func (obj *Test) EncodeRLP(_w io.Writer) error { + w := rlp.NewEncoderBuffer(_w) + _tmp0 := w.List() + w.WriteUint64(uint64(obj.A)) + w.WriteUint64(uint64(obj.B)) + w.WriteUint64(uint64(obj.C)) + w.WriteUint64(obj.D) + w.ListEnd(_tmp0) + return w.Flush() +} + +func (obj *Test) DecodeRLP(dec *rlp.Stream) error { + var _tmp0 Test + { + if _, err := dec.List(); err != nil { + return err + } + // A: + _tmp1, err := dec.Uint8() + if err != nil { + return err + } + _tmp0.A = _tmp1 + // B: + _tmp2, err := dec.Uint16() + if err != nil { + return err + } + _tmp0.B = _tmp2 + // C: + _tmp3, err := dec.Uint32() + if err != nil { + return err + } + _tmp0.C = _tmp3 + // D: + _tmp4, err := dec.Uint64() + if err != nil { + return err + } + _tmp0.D = _tmp4 + if err := dec.ListEnd(); err != nil { + return err + } + } + *obj = _tmp0 + return nil +} diff --git a/rlp/rlpgen/types.go b/rlp/rlpgen/types.go new file mode 100644 index 0000000000..ea7dc96d88 --- /dev/null +++ b/rlp/rlpgen/types.go @@ -0,0 +1,124 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package main + +import ( + "fmt" + "go/types" + "reflect" +) + +// typeReflectKind gives the reflect.Kind that represents typ. +func typeReflectKind(typ types.Type) reflect.Kind { + switch typ := typ.(type) { + case *types.Basic: + k := typ.Kind() + if k >= types.Bool && k <= types.Complex128 { + // value order matches for Bool..Complex128 + return reflect.Bool + reflect.Kind(k-types.Bool) + } + if k == types.String { + return reflect.String + } + if k == types.UnsafePointer { + return reflect.UnsafePointer + } + panic(fmt.Errorf("unhandled BasicKind %v", k)) + case *types.Array: + return reflect.Array + case *types.Chan: + return reflect.Chan + case *types.Interface: + return reflect.Interface + case *types.Map: + return reflect.Map + case *types.Pointer: + return reflect.Ptr + case *types.Signature: + return reflect.Func + case *types.Slice: + return reflect.Slice + case *types.Struct: + return reflect.Struct + default: + panic(fmt.Errorf("unhandled type %T", typ)) + } +} + +// nonZeroCheck returns the expression that checks whether 'v' is a non-zero value of type 'vtyp'. +func nonZeroCheck(v string, vtyp types.Type, qualify types.Qualifier) string { + // Resolve type name. + typ := resolveUnderlying(vtyp) + switch typ := typ.(type) { + case *types.Basic: + k := typ.Kind() + switch { + case k == types.Bool: + return v + case k >= types.Uint && k <= types.Complex128: + return fmt.Sprintf("%s != 0", v) + case k == types.String: + return fmt.Sprintf(`%s != ""`, v) + default: + panic(fmt.Errorf("unhandled BasicKind %v", k)) + } + case *types.Array, *types.Struct: + return fmt.Sprintf("%s != (%s{})", v, types.TypeString(vtyp, qualify)) + case *types.Interface, *types.Pointer, *types.Signature: + return fmt.Sprintf("%s != nil", v) + case *types.Slice, *types.Map: + return fmt.Sprintf("len(%s) > 0", v) + default: + panic(fmt.Errorf("unhandled type %T", typ)) + } +} + +// isBigInt checks whether 'typ' is "math/big".Int. +func isBigInt(typ types.Type) bool { + named, ok := typ.(*types.Named) + if !ok { + return false + } + name := named.Obj() + return name.Pkg().Path() == "math/big" && name.Name() == "Int" +} + +// isUint256 checks whether 'typ' is "github.com/holiman/uint256".Int. +func isUint256(typ types.Type) bool { + named, ok := typ.(*types.Named) + if !ok { + return false + } + name := named.Obj() + return name.Pkg().Path() == "github.com/holiman/uint256" && name.Name() == "Int" +} + +// isByte checks whether the underlying type of 'typ' is uint8. +func isByte(typ types.Type) bool { + basic, ok := resolveUnderlying(typ).(*types.Basic) + return ok && basic.Kind() == types.Uint8 +} + +func resolveUnderlying(typ types.Type) types.Type { + for { + t := typ.Underlying() + if t == typ { + return t + } + typ = t + } +} From ddf35f8c4f74ab1b2a801f4960428bb61f0b0526 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Fri, 16 Jun 2023 16:02:10 +0700 Subject: [PATCH 04/82] Update unit tests and benchmark --- rlp/decode_test.go | 520 ++++++++++++++++++++++++++++++++-- rlp/encbuffer_example_test.go | 45 +++ rlp/encode_test.go | 338 ++++++++++++++++++++-- rlp/encoder_example_test.go | 20 +- rlp/iterator_test.go | 59 ++++ rlp/raw_test.go | 160 ++++++++++- 6 files changed, 1074 insertions(+), 68 deletions(-) create mode 100644 rlp/encbuffer_example_test.go create mode 100644 rlp/iterator_test.go diff --git a/rlp/decode_test.go b/rlp/decode_test.go index 4d8abd0012..3ee237fb09 100644 --- a/rlp/decode_test.go +++ b/rlp/decode_test.go @@ -26,6 +26,10 @@ import ( "reflect" "strings" "testing" + + "github.com/tomochain/tomochain/common/math" + + "github.com/holiman/uint256" ) func TestStreamKind(t *testing.T) { @@ -284,6 +288,47 @@ func TestStreamRaw(t *testing.T) { } } +func TestStreamReadBytes(t *testing.T) { + tests := []struct { + input string + size int + err string + }{ + // kind List + {input: "C0", size: 1, err: "rlp: expected String or Byte"}, + // kind Byte + {input: "04", size: 0, err: "input value has wrong size 1, want 0"}, + {input: "04", size: 1}, + {input: "04", size: 2, err: "input value has wrong size 1, want 2"}, + // kind String + {input: "820102", size: 0, err: "input value has wrong size 2, want 0"}, + {input: "820102", size: 1, err: "input value has wrong size 2, want 1"}, + {input: "820102", size: 2}, + {input: "820102", size: 3, err: "input value has wrong size 2, want 3"}, + } + + for _, test := range tests { + test := test + name := fmt.Sprintf("input_%s/size_%d", test.input, test.size) + t.Run(name, func(t *testing.T) { + s := NewStream(bytes.NewReader(unhex(test.input)), 0) + b := make([]byte, test.size) + err := s.ReadBytes(b) + if test.err == "" { + if err != nil { + t.Errorf("unexpected error %q", err) + } + } else { + if err == nil { + t.Errorf("expected error, got nil") + } else if err.Error() != test.err { + t.Errorf("wrong error %q", err) + } + } + }) + } +} + func TestDecodeErrors(t *testing.T) { r := bytes.NewReader(nil) @@ -327,6 +372,15 @@ type recstruct struct { Child *recstruct `rlp:"nil"` } +type bigIntStruct struct { + I *big.Int + B string +} + +type invalidNilTag struct { + X []byte `rlp:"nil"` +} + type invalidTail1 struct { A uint `rlp:"tail"` B string @@ -347,19 +401,79 @@ type tailUint struct { Tail []uint `rlp:"tail"` } -var ( - veryBigInt = big.NewInt(0).Add( - big.NewInt(0).Lsh(big.NewInt(0xFFFFFFFFFFFFFF), 16), - big.NewInt(0xFFFF), - ) -) +type tailPrivateFields struct { + A uint + Tail []uint `rlp:"tail"` + x, y bool //lint:ignore U1000 unused fields required for testing purposes. +} + +type nilListUint struct { + X *uint `rlp:"nilList"` +} + +type nilStringSlice struct { + X *[]uint `rlp:"nilString"` +} + +type intField struct { + X int +} + +type optionalFields struct { + A uint + B uint `rlp:"optional"` + C uint `rlp:"optional"` +} + +type optionalAndTailField struct { + A uint + B uint `rlp:"optional"` + Tail []uint `rlp:"tail"` +} + +type optionalBigIntField struct { + A uint + B *big.Int `rlp:"optional"` +} + +type optionalPtrField struct { + A uint + B *[3]byte `rlp:"optional"` +} + +type nonOptionalPtrField struct { + A uint + B *[3]byte +} -type hasIgnoredField struct { +type multipleOptionalFields struct { + A *[3]byte `rlp:"optional"` + B *[3]byte `rlp:"optional"` +} + +type optionalPtrFieldNil struct { + A uint + B *[3]byte `rlp:"optional,nil"` +} + +type ignoredField struct { A uint B uint `rlp:"-"` C uint } +var ( + veryBigInt = new(big.Int).Add( + new(big.Int).Lsh(big.NewInt(0xFFFFFFFFFFFFFF), 16), + big.NewInt(0xFFFF), + ) + veryVeryBigInt = new(big.Int).Exp(veryBigInt, big.NewInt(8), nil) +) + +var ( + veryBigInt256, _ = uint256.FromBig(veryBigInt) +) + var decodeTests = []decodeTest{ // booleans {input: "01", ptr: new(bool), value: true}, @@ -428,12 +542,31 @@ var decodeTests = []decodeTest{ {input: "C0", ptr: new(string), error: "rlp: expected input string or byte for string"}, // big ints + {input: "80", ptr: new(*big.Int), value: big.NewInt(0)}, {input: "01", ptr: new(*big.Int), value: big.NewInt(1)}, {input: "89FFFFFFFFFFFFFFFFFF", ptr: new(*big.Int), value: veryBigInt}, + {input: "B848FFFFFFFFFFFFFFFFF800000000000000001BFFFFFFFFFFFFFFFFC8000000000000000045FFFFFFFFFFFFFFFFC800000000000000001BFFFFFFFFFFFFFFFFF8000000000000000001", ptr: new(*big.Int), value: veryVeryBigInt}, {input: "10", ptr: new(big.Int), value: *big.NewInt(16)}, // non-pointer also works + + // big int errors {input: "C0", ptr: new(*big.Int), error: "rlp: expected input string or byte for *big.Int"}, - {input: "820001", ptr: new(big.Int), error: "rlp: non-canonical integer (leading zero bytes) for *big.Int"}, - {input: "8105", ptr: new(big.Int), error: "rlp: non-canonical size information for *big.Int"}, + {input: "00", ptr: new(*big.Int), error: "rlp: non-canonical integer (leading zero bytes) for *big.Int"}, + {input: "820001", ptr: new(*big.Int), error: "rlp: non-canonical integer (leading zero bytes) for *big.Int"}, + {input: "8105", ptr: new(*big.Int), error: "rlp: non-canonical size information for *big.Int"}, + + // uint256 + {input: "80", ptr: new(*uint256.Int), value: uint256.NewInt(0)}, + {input: "01", ptr: new(*uint256.Int), value: uint256.NewInt(1)}, + {input: "88FFFFFFFFFFFFFFFF", ptr: new(*uint256.Int), value: uint256.NewInt(math.MaxUint64)}, + {input: "89FFFFFFFFFFFFFFFFFF", ptr: new(*uint256.Int), value: veryBigInt256}, + {input: "10", ptr: new(uint256.Int), value: *uint256.NewInt(16)}, // non-pointer also works + + // uint256 errors + {input: "C0", ptr: new(*uint256.Int), error: "rlp: expected input string or byte for *uint256.Int"}, + {input: "00", ptr: new(*uint256.Int), error: "rlp: non-canonical integer (leading zero bytes) for *uint256.Int"}, + {input: "820001", ptr: new(*uint256.Int), error: "rlp: non-canonical integer (leading zero bytes) for *uint256.Int"}, + {input: "8105", ptr: new(*uint256.Int), error: "rlp: non-canonical size information for *uint256.Int"}, + {input: "A1FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00", ptr: new(*uint256.Int), error: "rlp: value too large for uint256"}, // structs { @@ -446,6 +579,13 @@ var decodeTests = []decodeTest{ ptr: new(recstruct), value: recstruct{1, &recstruct{2, &recstruct{3, nil}}}, }, + { + // This checks that empty big.Int works correctly in struct context. It's easy to + // miss the update of s.kind for this case, so it needs its own test. + input: "C58083343434", + ptr: new(bigIntStruct), + value: bigIntStruct{new(big.Int), "444"}, + }, // struct errors { @@ -479,20 +619,20 @@ var decodeTests = []decodeTest{ error: "rlp: expected input string or byte for uint, decoding into (rlp.recstruct).Child.I", }, { - input: "C0", - ptr: new(invalidTail1), - error: "rlp: invalid struct tag \"tail\" for rlp.invalidTail1.A (must be on last field)", - }, - { - input: "C0", - ptr: new(invalidTail2), - error: "rlp: invalid struct tag \"tail\" for rlp.invalidTail2.B (field type is not slice)", + input: "C103", + ptr: new(intField), + error: "rlp: type int is not RLP-serializable (struct field rlp.intField.X)", }, { input: "C50102C20102", ptr: new(tailUint), error: "rlp: expected input string or byte for uint, decoding into (rlp.tailUint).Tail[1]", }, + { + input: "C0", + ptr: new(invalidNilTag), + error: `rlp: invalid struct tag "nil" for rlp.invalidNilTag.X (field is not a pointer)`, + }, // struct tag "tail" { @@ -510,12 +650,192 @@ var decodeTests = []decodeTest{ ptr: new(tailRaw), value: tailRaw{A: 1, Tail: []RawValue{}}, }, + { + input: "C3010203", + ptr: new(tailPrivateFields), + value: tailPrivateFields{A: 1, Tail: []uint{2, 3}}, + }, + { + input: "C0", + ptr: new(invalidTail1), + error: `rlp: invalid struct tag "tail" for rlp.invalidTail1.A (must be on last field)`, + }, + { + input: "C0", + ptr: new(invalidTail2), + error: `rlp: invalid struct tag "tail" for rlp.invalidTail2.B (field type is not slice)`, + }, // struct tag "-" { input: "C20102", - ptr: new(hasIgnoredField), - value: hasIgnoredField{A: 1, C: 2}, + ptr: new(ignoredField), + value: ignoredField{A: 1, C: 2}, + }, + + // struct tag "nilList" + { + input: "C180", + ptr: new(nilListUint), + error: "rlp: wrong kind of empty value (got String, want List) for *uint, decoding into (rlp.nilListUint).X", + }, + { + input: "C1C0", + ptr: new(nilListUint), + value: nilListUint{}, + }, + { + input: "C103", + ptr: new(nilListUint), + value: func() interface{} { + v := uint(3) + return nilListUint{X: &v} + }(), + }, + + // struct tag "nilString" + { + input: "C1C0", + ptr: new(nilStringSlice), + error: "rlp: wrong kind of empty value (got List, want String) for *[]uint, decoding into (rlp.nilStringSlice).X", + }, + { + input: "C180", + ptr: new(nilStringSlice), + value: nilStringSlice{}, + }, + { + input: "C2C103", + ptr: new(nilStringSlice), + value: nilStringSlice{X: &[]uint{3}}, + }, + + // struct tag "optional" + { + input: "C101", + ptr: new(optionalFields), + value: optionalFields{1, 0, 0}, + }, + { + input: "C20102", + ptr: new(optionalFields), + value: optionalFields{1, 2, 0}, + }, + { + input: "C3010203", + ptr: new(optionalFields), + value: optionalFields{1, 2, 3}, + }, + { + input: "C401020304", + ptr: new(optionalFields), + error: "rlp: input list has too many elements for rlp.optionalFields", + }, + { + input: "C101", + ptr: new(optionalAndTailField), + value: optionalAndTailField{A: 1}, + }, + { + input: "C20102", + ptr: new(optionalAndTailField), + value: optionalAndTailField{A: 1, B: 2, Tail: []uint{}}, + }, + { + input: "C401020304", + ptr: new(optionalAndTailField), + value: optionalAndTailField{A: 1, B: 2, Tail: []uint{3, 4}}, + }, + { + input: "C101", + ptr: new(optionalBigIntField), + value: optionalBigIntField{A: 1, B: nil}, + }, + { + input: "C20102", + ptr: new(optionalBigIntField), + value: optionalBigIntField{A: 1, B: big.NewInt(2)}, + }, + { + input: "C101", + ptr: new(optionalPtrField), + value: optionalPtrField{A: 1}, + }, + { + input: "C20180", // not accepted because "optional" doesn't enable "nil" + ptr: new(optionalPtrField), + error: "rlp: input string too short for [3]uint8, decoding into (rlp.optionalPtrField).B", + }, + { + input: "C20102", + ptr: new(optionalPtrField), + error: "rlp: input string too short for [3]uint8, decoding into (rlp.optionalPtrField).B", + }, + { + input: "C50183010203", + ptr: new(optionalPtrField), + value: optionalPtrField{A: 1, B: &[3]byte{1, 2, 3}}, + }, + { + // all optional fields nil + input: "C0", + ptr: new(multipleOptionalFields), + value: multipleOptionalFields{A: nil, B: nil}, + }, + { + // all optional fields set + input: "C88301020383010203", + ptr: new(multipleOptionalFields), + value: multipleOptionalFields{A: &[3]byte{1, 2, 3}, B: &[3]byte{1, 2, 3}}, + }, + { + // nil optional field appears before a non-nil one + input: "C58083010203", + ptr: new(multipleOptionalFields), + error: "rlp: input string too short for [3]uint8, decoding into (rlp.multipleOptionalFields).A", + }, + { + // decode a nil ptr into a ptr that is not nil or not optional + input: "C20180", + ptr: new(nonOptionalPtrField), + error: "rlp: input string too short for [3]uint8, decoding into (rlp.nonOptionalPtrField).B", + }, + { + input: "C101", + ptr: new(optionalPtrFieldNil), + value: optionalPtrFieldNil{A: 1}, + }, + { + input: "C20180", // accepted because "nil" tag allows empty input + ptr: new(optionalPtrFieldNil), + value: optionalPtrFieldNil{A: 1}, + }, + { + input: "C20102", + ptr: new(optionalPtrFieldNil), + error: "rlp: input string too short for [3]uint8, decoding into (rlp.optionalPtrFieldNil).B", + }, + + // struct tag "optional" field clearing + { + input: "C101", + ptr: &optionalFields{A: 9, B: 8, C: 7}, + value: optionalFields{A: 1, B: 0, C: 0}, + }, + { + input: "C20102", + ptr: &optionalFields{A: 9, B: 8, C: 7}, + value: optionalFields{A: 1, B: 2, C: 0}, + }, + { + input: "C20102", + ptr: &optionalAndTailField{A: 9, B: 8, Tail: []uint{7, 6, 5}}, + value: optionalAndTailField{A: 1, B: 2, Tail: []uint{}}, + }, + { + input: "C101", + ptr: &optionalPtrField{A: 9, B: &[3]byte{8, 7, 6}}, + value: optionalPtrField{A: 1}, }, // RawValue @@ -591,6 +911,26 @@ func TestDecodeWithByteReader(t *testing.T) { }) } +func testDecodeWithEncReader(t *testing.T, n int) { + s := strings.Repeat("0", n) + _, r, _ := EncodeToReader(s) + var decoded string + err := Decode(r, &decoded) + if err != nil { + t.Errorf("Unexpected decode error with n=%v: %v", n, err) + } + if decoded != s { + t.Errorf("Decode mismatch with n=%v", n) + } +} + +// This is a regression test checking that decoding from encReader +// works for RLP values of size 8192 bytes or more. +func TestDecodeWithEncReader(t *testing.T) { + testDecodeWithEncReader(t, 8188) // length with header is 8191 + testDecodeWithEncReader(t, 8189) // length with header is 8192 +} + // plainReader reads from a byte slice but does not // implement ReadByte. It is also not recognized by the // size validation. This is useful to test how the decoder @@ -661,6 +1001,22 @@ func TestDecodeDecoder(t *testing.T) { } } +func TestDecodeDecoderNilPointer(t *testing.T) { + var s struct { + T1 *testDecoder `rlp:"nil"` + T2 *testDecoder + } + if err := Decode(bytes.NewReader(unhex("C2C002")), &s); err != nil { + t.Fatalf("Decode error: %v", err) + } + if s.T1 != nil { + t.Errorf("decoder T1 allocated for empty input (called: %v)", s.T1.called) + } + if s.T2 == nil || !s.T2.called { + t.Errorf("decoder T2 not allocated/called") + } +} + type byteDecoder byte func (bd *byteDecoder) DecodeRLP(s *Stream) error { @@ -691,13 +1047,66 @@ func TestDecoderInByteSlice(t *testing.T) { } } +type unencodableDecoder func() + +func (f *unencodableDecoder) DecodeRLP(s *Stream) error { + if _, err := s.List(); err != nil { + return err + } + if err := s.ListEnd(); err != nil { + return err + } + *f = func() {} + return nil +} + +func TestDecoderFunc(t *testing.T) { + var x func() + if err := DecodeBytes([]byte{0xC0}, (*unencodableDecoder)(&x)); err != nil { + t.Fatal(err) + } + x() +} + +// This tests the validity checks for fields with struct tag "optional". +func TestInvalidOptionalField(t *testing.T) { + type ( + invalid1 struct { + A uint `rlp:"optional"` + B uint + } + invalid2 struct { + T []uint `rlp:"tail,optional"` + } + invalid3 struct { + T []uint `rlp:"optional,tail"` + } + ) + + tests := []struct { + v interface{} + err string + }{ + {v: new(invalid1), err: `rlp: invalid struct tag "" for rlp.invalid1.B (must be optional because preceding field "A" is optional)`}, + {v: new(invalid2), err: `rlp: invalid struct tag "optional" for rlp.invalid2.T (also has "tail" tag)`}, + {v: new(invalid3), err: `rlp: invalid struct tag "tail" for rlp.invalid3.T (also has "optional" tag)`}, + } + for _, test := range tests { + err := DecodeBytes(unhex("C20102"), test.v) + if err == nil { + t.Errorf("no error for %T", test.v) + } else if err.Error() != test.err { + t.Errorf("wrong error for %T: %v", test.v, err.Error()) + } + } +} + func ExampleDecode() { input, _ := hex.DecodeString("C90A1486666F6F626172") type example struct { - A, B uint - private uint // private fields are ignored - String string + A, B uint + String string } var s example @@ -708,7 +1117,7 @@ func ExampleDecode() { fmt.Printf("Decoded value: %#v\n", s) } // Output: - // Decoded value: rlp.example{A:0xa, B:0x14, private:0x0, String:"foobar"} + // Decoded value: rlp.example{A:0xa, B:0x14, String:"foobar"} } func ExampleDecode_structTagNil() { @@ -768,7 +1177,7 @@ func ExampleStream() { // [102 111 111 98 97 114] } -func BenchmarkDecode(b *testing.B) { +func BenchmarkDecodeUints(b *testing.B) { enc := encodeTestSlice(90000) b.SetBytes(int64(len(enc))) b.ReportAllocs() @@ -783,7 +1192,7 @@ func BenchmarkDecode(b *testing.B) { } } -func BenchmarkDecodeIntSliceReuse(b *testing.B) { +func BenchmarkDecodeUintsReused(b *testing.B) { enc := encodeTestSlice(100000) b.SetBytes(int64(len(enc))) b.ReportAllocs() @@ -798,6 +1207,65 @@ func BenchmarkDecodeIntSliceReuse(b *testing.B) { } } +func BenchmarkDecodeByteArrayStruct(b *testing.B) { + enc, err := EncodeToBytes(&byteArrayStruct{}) + if err != nil { + b.Fatal(err) + } + b.SetBytes(int64(len(enc))) + b.ReportAllocs() + b.ResetTimer() + + var out byteArrayStruct + for i := 0; i < b.N; i++ { + if err := DecodeBytes(enc, &out); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkDecodeBigInts(b *testing.B) { + ints := make([]*big.Int, 200) + for i := range ints { + ints[i] = math.BigPow(2, int64(i)) + } + enc, err := EncodeToBytes(ints) + if err != nil { + b.Fatal(err) + } + b.SetBytes(int64(len(enc))) + b.ReportAllocs() + b.ResetTimer() + + var out []*big.Int + for i := 0; i < b.N; i++ { + if err := DecodeBytes(enc, &out); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkDecodeU256Ints(b *testing.B) { + ints := make([]*uint256.Int, 200) + for i := range ints { + ints[i], _ = uint256.FromBig(math.BigPow(2, int64(i))) + } + enc, err := EncodeToBytes(ints) + if err != nil { + b.Fatal(err) + } + b.SetBytes(int64(len(enc))) + b.ReportAllocs() + b.ResetTimer() + + var out []*uint256.Int + for i := 0; i < b.N; i++ { + if err := DecodeBytes(enc, &out); err != nil { + b.Fatal(err) + } + } +} + func encodeTestSlice(n uint) []byte { s := make([]uint, n) for i := uint(0); i < n; i++ { @@ -811,7 +1279,7 @@ func encodeTestSlice(n uint) []byte { } func unhex(str string) []byte { - b, err := hex.DecodeString(strings.Replace(str, " ", "", -1)) + b, err := hex.DecodeString(strings.ReplaceAll(str, " ", "")) if err != nil { panic(fmt.Sprintf("invalid hex string: %q", str)) } diff --git a/rlp/encbuffer_example_test.go b/rlp/encbuffer_example_test.go new file mode 100644 index 0000000000..c41de60f02 --- /dev/null +++ b/rlp/encbuffer_example_test.go @@ -0,0 +1,45 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rlp_test + +import ( + "bytes" + "fmt" + + "github.com/tomochain/tomochain/rlp" +) + +func ExampleEncoderBuffer() { + var w bytes.Buffer + + // Encode [4, [5, 6]] to w. + buf := rlp.NewEncoderBuffer(&w) + l1 := buf.List() + buf.WriteUint64(4) + l2 := buf.List() + buf.WriteUint64(5) + buf.WriteUint64(6) + buf.ListEnd(l2) + buf.ListEnd(l1) + + if err := buf.Flush(); err != nil { + panic(err) + } + fmt.Printf("%X\n", w.Bytes()) + // Output: + // C404C20506 +} diff --git a/rlp/encode_test.go b/rlp/encode_test.go index 827960f7c1..7b8775c12b 100644 --- a/rlp/encode_test.go +++ b/rlp/encode_test.go @@ -21,10 +21,14 @@ import ( "errors" "fmt" "io" - "io/ioutil" "math/big" + "runtime" "sync" "testing" + + "github.com/tomochain/tomochain/common/math" + + "github.com/holiman/uint256" ) type testEncoder struct { @@ -33,12 +37,19 @@ type testEncoder struct { func (e *testEncoder) EncodeRLP(w io.Writer) error { if e == nil { - w.Write([]byte{0, 0, 0, 0}) - } else if e.err != nil { + panic("EncodeRLP called on nil value") + } + if e.err != nil { return e.err - } else { - w.Write([]byte{0, 1, 0, 1, 0, 1, 0, 1, 0, 1}) } + w.Write([]byte{0, 1, 0, 1, 0, 1, 0, 1, 0, 1}) + return nil +} + +type testEncoderValueMethod struct{} + +func (e testEncoderValueMethod) EncodeRLP(w io.Writer) error { + w.Write([]byte{0xFA, 0xFE, 0xF0}) return nil } @@ -49,6 +60,13 @@ func (e byteEncoder) EncodeRLP(w io.Writer) error { return nil } +type undecodableEncoder func() + +func (f undecodableEncoder) EncodeRLP(w io.Writer) error { + w.Write([]byte{0xF5, 0xF5, 0xF5}) + return nil +} + type encodableReader struct { A, B uint } @@ -103,35 +121,95 @@ var encTests = []encTest{ {val: big.NewInt(0xFFFFFFFFFFFF), output: "86FFFFFFFFFFFF"}, {val: big.NewInt(0xFFFFFFFFFFFFFF), output: "87FFFFFFFFFFFFFF"}, { - val: big.NewInt(0).SetBytes(unhex("102030405060708090A0B0C0D0E0F2")), + val: new(big.Int).SetBytes(unhex("102030405060708090A0B0C0D0E0F2")), output: "8F102030405060708090A0B0C0D0E0F2", }, { - val: big.NewInt(0).SetBytes(unhex("0100020003000400050006000700080009000A000B000C000D000E01")), + val: new(big.Int).SetBytes(unhex("0100020003000400050006000700080009000A000B000C000D000E01")), output: "9C0100020003000400050006000700080009000A000B000C000D000E01", }, { - val: big.NewInt(0).SetBytes(unhex("010000000000000000000000000000000000000000000000000000000000000000")), + val: new(big.Int).SetBytes(unhex("010000000000000000000000000000000000000000000000000000000000000000")), output: "A1010000000000000000000000000000000000000000000000000000000000000000", }, + { + val: veryBigInt, + output: "89FFFFFFFFFFFFFFFFFF", + }, + { + val: veryVeryBigInt, + output: "B848FFFFFFFFFFFFFFFFF800000000000000001BFFFFFFFFFFFFFFFFC8000000000000000045FFFFFFFFFFFFFFFFC800000000000000001BFFFFFFFFFFFFFFFFF8000000000000000001", + }, // non-pointer big.Int {val: *big.NewInt(0), output: "80"}, {val: *big.NewInt(0xFFFFFF), output: "83FFFFFF"}, // negative ints are not supported - {val: big.NewInt(-1), error: "rlp: cannot encode negative *big.Int"}, - - // byte slices, strings + {val: big.NewInt(-1), error: "rlp: cannot encode negative big.Int"}, + {val: *big.NewInt(-1), error: "rlp: cannot encode negative big.Int"}, + + // uint256 + {val: uint256.NewInt(0), output: "80"}, + {val: uint256.NewInt(1), output: "01"}, + {val: uint256.NewInt(127), output: "7F"}, + {val: uint256.NewInt(128), output: "8180"}, + {val: uint256.NewInt(256), output: "820100"}, + {val: uint256.NewInt(1024), output: "820400"}, + {val: uint256.NewInt(0xFFFFFF), output: "83FFFFFF"}, + {val: uint256.NewInt(0xFFFFFFFF), output: "84FFFFFFFF"}, + {val: uint256.NewInt(0xFFFFFFFFFF), output: "85FFFFFFFFFF"}, + {val: uint256.NewInt(0xFFFFFFFFFFFF), output: "86FFFFFFFFFFFF"}, + {val: uint256.NewInt(0xFFFFFFFFFFFFFF), output: "87FFFFFFFFFFFFFF"}, + { + val: new(uint256.Int).SetBytes(unhex("102030405060708090A0B0C0D0E0F2")), + output: "8F102030405060708090A0B0C0D0E0F2", + }, + { + val: new(uint256.Int).SetBytes(unhex("0100020003000400050006000700080009000A000B000C000D000E01")), + output: "9C0100020003000400050006000700080009000A000B000C000D000E01", + }, + // non-pointer uint256.Int + {val: *uint256.NewInt(0), output: "80"}, + {val: *uint256.NewInt(0xFFFFFF), output: "83FFFFFF"}, + + // byte arrays + {val: [0]byte{}, output: "80"}, + {val: [1]byte{0}, output: "00"}, + {val: [1]byte{1}, output: "01"}, + {val: [1]byte{0x7F}, output: "7F"}, + {val: [1]byte{0x80}, output: "8180"}, + {val: [1]byte{0xFF}, output: "81FF"}, + {val: [3]byte{1, 2, 3}, output: "83010203"}, + {val: [57]byte{1, 2, 3}, output: "B839010203000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}, + + // named byte type arrays + {val: [0]namedByteType{}, output: "80"}, + {val: [1]namedByteType{0}, output: "00"}, + {val: [1]namedByteType{1}, output: "01"}, + {val: [1]namedByteType{0x7F}, output: "7F"}, + {val: [1]namedByteType{0x80}, output: "8180"}, + {val: [1]namedByteType{0xFF}, output: "81FF"}, + {val: [3]namedByteType{1, 2, 3}, output: "83010203"}, + {val: [57]namedByteType{1, 2, 3}, output: "B839010203000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}, + + // byte slices {val: []byte{}, output: "80"}, + {val: []byte{0}, output: "00"}, {val: []byte{0x7E}, output: "7E"}, {val: []byte{0x7F}, output: "7F"}, {val: []byte{0x80}, output: "8180"}, {val: []byte{1, 2, 3}, output: "83010203"}, + // named byte type slices + {val: []namedByteType{}, output: "80"}, + {val: []namedByteType{0}, output: "00"}, + {val: []namedByteType{0x7E}, output: "7E"}, + {val: []namedByteType{0x7F}, output: "7F"}, + {val: []namedByteType{0x80}, output: "8180"}, {val: []namedByteType{1, 2, 3}, output: "83010203"}, - {val: [...]namedByteType{1, 2, 3}, output: "83010203"}, + // strings {val: "", output: "80"}, {val: "\x7E", output: "7E"}, {val: "\x7F", output: "7F"}, @@ -204,6 +282,12 @@ var encTests = []encTest{ output: "F90200CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376", }, + // Non-byte arrays are encoded as lists. + // Note that it is important to test [4]uint64 specifically, + // because that's the underlying type of uint256.Int. + {val: [4]uint32{1, 2, 3, 4}, output: "C401020304"}, + {val: [4]uint64{1, 2, 3, 4}, output: "C401020304"}, + // RawValue {val: RawValue(unhex("01")), output: "01"}, {val: RawValue(unhex("82FFFF")), output: "82FFFF"}, @@ -214,11 +298,34 @@ var encTests = []encTest{ {val: simplestruct{A: 3, B: "foo"}, output: "C50383666F6F"}, {val: &recstruct{5, nil}, output: "C205C0"}, {val: &recstruct{5, &recstruct{4, &recstruct{3, nil}}}, output: "C605C404C203C0"}, + {val: &intField{X: 3}, error: "rlp: type int is not RLP-serializable (struct field rlp.intField.X)"}, + + // struct tag "-" + {val: &ignoredField{A: 1, B: 2, C: 3}, output: "C20103"}, + + // struct tag "tail" {val: &tailRaw{A: 1, Tail: []RawValue{unhex("02"), unhex("03")}}, output: "C3010203"}, {val: &tailRaw{A: 1, Tail: []RawValue{unhex("02")}}, output: "C20102"}, {val: &tailRaw{A: 1, Tail: []RawValue{}}, output: "C101"}, {val: &tailRaw{A: 1, Tail: nil}, output: "C101"}, - {val: &hasIgnoredField{A: 1, B: 2, C: 3}, output: "C20103"}, + + // struct tag "optional" + {val: &optionalFields{}, output: "C180"}, + {val: &optionalFields{A: 1}, output: "C101"}, + {val: &optionalFields{A: 1, B: 2}, output: "C20102"}, + {val: &optionalFields{A: 1, B: 2, C: 3}, output: "C3010203"}, + {val: &optionalFields{A: 1, B: 0, C: 3}, output: "C3018003"}, + {val: &optionalAndTailField{A: 1}, output: "C101"}, + {val: &optionalAndTailField{A: 1, B: 2}, output: "C20102"}, + {val: &optionalAndTailField{A: 1, Tail: []uint{5, 6}}, output: "C401800506"}, + {val: &optionalAndTailField{A: 1, Tail: []uint{5, 6}}, output: "C401800506"}, + {val: &optionalBigIntField{A: 1}, output: "C101"}, + {val: &optionalPtrField{A: 1}, output: "C101"}, + {val: &optionalPtrFieldNil{A: 1}, output: "C101"}, + {val: &multipleOptionalFields{A: nil, B: nil}, output: "C0"}, + {val: &multipleOptionalFields{A: &[3]byte{1, 2, 3}, B: &[3]byte{1, 2, 3}}, output: "C88301020383010203"}, + {val: &multipleOptionalFields{A: nil, B: &[3]byte{1, 2, 3}}, output: "C58083010203"}, // encodes without error but decode will fail + {val: &nonOptionalPtrField{A: 1}, output: "C20180"}, // encodes without error but decode will fail // nil {val: (*uint)(nil), output: "80"}, @@ -226,26 +333,73 @@ var encTests = []encTest{ {val: (*[]byte)(nil), output: "80"}, {val: (*[10]byte)(nil), output: "80"}, {val: (*big.Int)(nil), output: "80"}, + {val: (*uint256.Int)(nil), output: "80"}, {val: (*[]string)(nil), output: "C0"}, {val: (*[10]string)(nil), output: "C0"}, {val: (*[]interface{})(nil), output: "C0"}, {val: (*[]struct{ uint })(nil), output: "C0"}, {val: (*interface{})(nil), output: "C0"}, + // nil struct fields + { + val: struct { + X *[]byte + }{}, + output: "C180", + }, + { + val: struct { + X *[2]byte + }{}, + output: "C180", + }, + { + val: struct { + X *uint64 + }{}, + output: "C180", + }, + { + val: struct { + X *uint64 `rlp:"nilList"` + }{}, + output: "C1C0", + }, + { + val: struct { + X *[]uint64 + }{}, + output: "C1C0", + }, + { + val: struct { + X *[]uint64 `rlp:"nilString"` + }{}, + output: "C180", + }, + // interfaces {val: []io.Reader{reader}, output: "C3C20102"}, // the contained value is a struct // Encoder - {val: (*testEncoder)(nil), output: "00000000"}, + {val: (*testEncoder)(nil), output: "C0"}, {val: &testEncoder{}, output: "00010001000100010001"}, {val: &testEncoder{errors.New("test error")}, error: "test error"}, - // verify that pointer method testEncoder.EncodeRLP is called for + {val: struct{ E testEncoderValueMethod }{}, output: "C3FAFEF0"}, + {val: struct{ E *testEncoderValueMethod }{}, output: "C1C0"}, + + // Verify that the Encoder interface works for unsupported types like func(). + {val: undecodableEncoder(func() {}), output: "F5F5F5"}, + + // Verify that pointer method testEncoder.EncodeRLP is called for // addressable non-pointer values. {val: &struct{ TE testEncoder }{testEncoder{}}, output: "CA00010001000100010001"}, {val: &struct{ TE testEncoder }{testEncoder{errors.New("test error")}}, error: "test error"}, - // verify the error for non-addressable non-pointer Encoder - {val: testEncoder{}, error: "rlp: game over: unadressable value of type rlp.testEncoder, EncodeRLP is pointer method"}, - // verify the special case for []byte + + // Verify the error for non-addressable non-pointer Encoder. + {val: testEncoder{}, error: "rlp: unadressable value of type rlp.testEncoder, EncodeRLP is pointer method"}, + + // Verify Encoder takes precedence over []byte. {val: []byteEncoder{0, 1, 2, 3, 4}, output: "C5C0C0C0C0C0"}, } @@ -281,13 +435,28 @@ func TestEncodeToBytes(t *testing.T) { runEncTests(t, EncodeToBytes) } +func TestEncodeAppendToBytes(t *testing.T) { + buffer := make([]byte, 20) + runEncTests(t, func(val interface{}) ([]byte, error) { + w := NewEncoderBuffer(nil) + defer w.Flush() + + err := Encode(w, val) + if err != nil { + return nil, err + } + output := w.AppendToBytes(buffer[:0]) + return output, nil + }) +} + func TestEncodeToReader(t *testing.T) { runEncTests(t, func(val interface{}) ([]byte, error) { _, r, err := EncodeToReader(val) if err != nil { return nil, err } - return ioutil.ReadAll(r) + return io.ReadAll(r) }) } @@ -328,7 +497,7 @@ func TestEncodeToReaderReturnToPool(t *testing.T) { go func() { for i := 0; i < 1000; i++ { _, r, _ := EncodeToReader("foo") - ioutil.ReadAll(r) + io.ReadAll(r) r.Read(buf) r.Read(buf) r.Read(buf) @@ -339,3 +508,132 @@ func TestEncodeToReaderReturnToPool(t *testing.T) { } wg.Wait() } + +var sink interface{} + +func BenchmarkIntsize(b *testing.B) { + for i := 0; i < b.N; i++ { + sink = intsize(0x12345678) + } +} + +func BenchmarkPutint(b *testing.B) { + buf := make([]byte, 8) + for i := 0; i < b.N; i++ { + putint(buf, 0x12345678) + sink = buf + } +} + +func BenchmarkEncodeBigInts(b *testing.B) { + ints := make([]*big.Int, 200) + for i := range ints { + ints[i] = math.BigPow(2, int64(i)) + } + out := bytes.NewBuffer(make([]byte, 0, 4096)) + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + out.Reset() + if err := Encode(out, ints); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkEncodeU256Ints(b *testing.B) { + ints := make([]*uint256.Int, 200) + for i := range ints { + ints[i], _ = uint256.FromBig(math.BigPow(2, int64(i))) + } + out := bytes.NewBuffer(make([]byte, 0, 4096)) + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + out.Reset() + if err := Encode(out, ints); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkEncodeConcurrentInterface(b *testing.B) { + type struct1 struct { + A string + B *big.Int + C [20]byte + } + value := []interface{}{ + uint(999), + &struct1{A: "hello", B: big.NewInt(0xFFFFFFFF)}, + [10]byte{1, 2, 3, 4, 5, 6}, + []string{"yeah", "yeah", "yeah"}, + } + + var wg sync.WaitGroup + for cpu := 0; cpu < runtime.NumCPU(); cpu++ { + wg.Add(1) + go func() { + defer wg.Done() + + var buffer bytes.Buffer + for i := 0; i < b.N; i++ { + buffer.Reset() + err := Encode(&buffer, value) + if err != nil { + panic(err) + } + } + }() + } + wg.Wait() +} + +type byteArrayStruct struct { + A [20]byte + B [32]byte + C [32]byte +} + +func BenchmarkEncodeByteArrayStruct(b *testing.B) { + var out bytes.Buffer + var value byteArrayStruct + + b.ReportAllocs() + for i := 0; i < b.N; i++ { + out.Reset() + if err := Encode(&out, &value); err != nil { + b.Fatal(err) + } + } +} + +type structSliceElem struct { + X uint64 + Y uint64 + Z uint64 +} + +type structPtrSlice []*structSliceElem + +func BenchmarkEncodeStructPtrSlice(b *testing.B) { + var out bytes.Buffer + var value = structPtrSlice{ + &structSliceElem{1, 1, 1}, + &structSliceElem{2, 2, 2}, + &structSliceElem{3, 3, 3}, + &structSliceElem{5, 5, 5}, + &structSliceElem{6, 6, 6}, + &structSliceElem{7, 7, 7}, + } + + b.ReportAllocs() + for i := 0; i < b.N; i++ { + out.Reset() + if err := Encode(&out, &value); err != nil { + b.Fatal(err) + } + } +} diff --git a/rlp/encoder_example_test.go b/rlp/encoder_example_test.go index 1cffa241c2..6291bfafe5 100644 --- a/rlp/encoder_example_test.go +++ b/rlp/encoder_example_test.go @@ -14,11 +14,13 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package rlp +package rlp_test import ( "fmt" "io" + + "github.com/tomochain/tomochain/rlp" ) type MyCoolType struct { @@ -28,27 +30,19 @@ type MyCoolType struct { // EncodeRLP writes x as RLP list [a, b] that omits the Name field. func (x *MyCoolType) EncodeRLP(w io.Writer) (err error) { - // Note: the receiver can be a nil pointer. This allows you to - // control the encoding of nil, but it also means that you have to - // check for a nil receiver. - if x == nil { - err = Encode(w, []uint{0, 0}) - } else { - err = Encode(w, []uint{x.a, x.b}) - } - return err + return rlp.Encode(w, []uint{x.a, x.b}) } func ExampleEncoder() { var t *MyCoolType // t is nil pointer to MyCoolType - bytes, _ := EncodeToBytes(t) + bytes, _ := rlp.EncodeToBytes(t) fmt.Printf("%v → %X\n", t, bytes) t = &MyCoolType{Name: "foobar", a: 5, b: 6} - bytes, _ = EncodeToBytes(t) + bytes, _ = rlp.EncodeToBytes(t) fmt.Printf("%v → %X\n", t, bytes) // Output: - // → C28080 + // → C0 // &{foobar 5 6} → C20506 } diff --git a/rlp/iterator_test.go b/rlp/iterator_test.go new file mode 100644 index 0000000000..87c11bdbae --- /dev/null +++ b/rlp/iterator_test.go @@ -0,0 +1,59 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rlp + +import ( + "testing" + + "github.com/tomochain/tomochain/common/hexutil" +) + +// TestIterator tests some basic things about the ListIterator. A more +// comprehensive test can be found in core/rlp_test.go, where we can +// use both types and rlp without dependency cycles +func TestIterator(t *testing.T) { + bodyRlpHex := "0xf902cbf8d6f869800182c35094000000000000000000000000000000000000aaaa808a000000000000000000001ba01025c66fad28b4ce3370222624d952c35529e602af7cbe04f667371f61b0e3b3a00ab8813514d1217059748fd903288ace1b4001a4bc5fbde2790debdc8167de2ff869010182c35094000000000000000000000000000000000000aaaa808a000000000000000000001ca05ac4cf1d19be06f3742c21df6c49a7e929ceb3dbaf6a09f3cfb56ff6828bd9a7a06875970133a35e63ac06d360aa166d228cc013e9b96e0a2cae7f55b22e1ee2e8f901f0f901eda0c75448377c0e426b8017b23c5f77379ecf69abc1d5c224284ad3ba1c46c59adaa00000000000000000000000000000000000000000000000000000000000000000940000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000b9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000808080808080a00000000000000000000000000000000000000000000000000000000000000000880000000000000000" + bodyRlp := hexutil.MustDecode(bodyRlpHex) + + it, err := NewListIterator(bodyRlp) + if err != nil { + t.Fatal(err) + } + // Check that txs exist + if !it.Next() { + t.Fatal("expected two elems, got zero") + } + txs := it.Value() + // Check that uncles exist + if !it.Next() { + t.Fatal("expected two elems, got one") + } + txit, err := NewListIterator(txs) + if err != nil { + t.Fatal(err) + } + var i = 0 + for txit.Next() { + if txit.err != nil { + t.Fatal(txit.err) + } + i++ + } + if exp := 2; i != exp { + t.Errorf("count wrong, expected %d got %d", i, exp) + } +} diff --git a/rlp/raw_test.go b/rlp/raw_test.go index 2aad042100..7b3255eca3 100644 --- a/rlp/raw_test.go +++ b/rlp/raw_test.go @@ -18,9 +18,10 @@ package rlp import ( "bytes" + "errors" "io" - "reflect" "testing" + "testing/quick" ) func TestCountValues(t *testing.T) { @@ -53,21 +54,84 @@ func TestCountValues(t *testing.T) { if count != test.count { t.Errorf("test %d: count mismatch, got %d want %d\ninput: %s", i, count, test.count, test.input) } - if !reflect.DeepEqual(err, test.err) { + if !errors.Is(err, test.err) { t.Errorf("test %d: err mismatch, got %q want %q\ninput: %s", i, err, test.err, test.input) } } } -func TestSplitTypes(t *testing.T) { - if _, _, err := SplitString(unhex("C100")); err != ErrExpectedString { - t.Errorf("SplitString returned %q, want %q", err, ErrExpectedString) +func TestSplitString(t *testing.T) { + for i, test := range []string{ + "C0", + "C100", + "C3010203", + "C88363617483646F67", + "F8384C6F72656D20697073756D20646F6C6F722073697420616D65742C20636F6E7365637465747572206164697069736963696E6720656C6974", + } { + if _, _, err := SplitString(unhex(test)); !errors.Is(err, ErrExpectedString) { + t.Errorf("test %d: error mismatch: have %q, want %q", i, err, ErrExpectedString) + } + } +} + +func TestSplitList(t *testing.T) { + for i, test := range []string{ + "80", + "00", + "01", + "8180", + "81FF", + "820400", + "83636174", + "83646F67", + "B8384C6F72656D20697073756D20646F6C6F722073697420616D65742C20636F6E7365637465747572206164697069736963696E6720656C6974", + } { + if _, _, err := SplitList(unhex(test)); !errors.Is(err, ErrExpectedList) { + t.Errorf("test %d: error mismatch: have %q, want %q", i, err, ErrExpectedList) + } } - if _, _, err := SplitList(unhex("01")); err != ErrExpectedList { - t.Errorf("SplitString returned %q, want %q", err, ErrExpectedList) +} + +func TestSplitUint64(t *testing.T) { + tests := []struct { + input string + val uint64 + rest string + err error + }{ + {"01", 1, "", nil}, + {"7FFF", 0x7F, "FF", nil}, + {"80FF", 0, "FF", nil}, + {"81FAFF", 0xFA, "FF", nil}, + {"82FAFAFF", 0xFAFA, "FF", nil}, + {"83FAFAFAFF", 0xFAFAFA, "FF", nil}, + {"84FAFAFAFAFF", 0xFAFAFAFA, "FF", nil}, + {"85FAFAFAFAFAFF", 0xFAFAFAFAFA, "FF", nil}, + {"86FAFAFAFAFAFAFF", 0xFAFAFAFAFAFA, "FF", nil}, + {"87FAFAFAFAFAFAFAFF", 0xFAFAFAFAFAFAFA, "FF", nil}, + {"88FAFAFAFAFAFAFAFAFF", 0xFAFAFAFAFAFAFAFA, "FF", nil}, + + // errors + {"", 0, "", io.ErrUnexpectedEOF}, + {"00", 0, "00", ErrCanonInt}, + {"81", 0, "81", ErrValueTooLarge}, + {"8100", 0, "8100", ErrCanonSize}, + {"8200FF", 0, "8200FF", ErrCanonInt}, + {"8103FF", 0, "8103FF", ErrCanonSize}, + {"89FAFAFAFAFAFAFAFAFAFF", 0, "89FAFAFAFAFAFAFAFAFAFF", errUintOverflow}, } - if _, _, err := SplitList(unhex("81FF")); err != ErrExpectedList { - t.Errorf("SplitString returned %q, want %q", err, ErrExpectedList) + + for i, test := range tests { + val, rest, err := SplitUint64(unhex(test.input)) + if val != test.val { + t.Errorf("test %d: val mismatch: got %x, want %x (input %q)", i, val, test.val, test.input) + } + if !bytes.Equal(rest, unhex(test.rest)) { + t.Errorf("test %d: rest mismatch: got %x, want %s (input %q)", i, rest, test.rest, test.input) + } + if err != test.err { + t.Errorf("test %d: error mismatch: got %q, want %q", i, err, test.err) + } } } @@ -78,7 +142,9 @@ func TestSplit(t *testing.T) { val, rest string err error }{ + {input: "00FFFF", kind: Byte, val: "00", rest: "FFFF"}, {input: "01FFFF", kind: Byte, val: "01", rest: "FFFF"}, + {input: "7FFFFF", kind: Byte, val: "7F", rest: "FFFF"}, {input: "80FFFF", kind: String, val: "", rest: "FFFF"}, {input: "C3010203", kind: List, val: "010203"}, @@ -194,3 +260,79 @@ func TestReadSize(t *testing.T) { } } } + +func TestAppendUint64(t *testing.T) { + tests := []struct { + input uint64 + slice []byte + output string + }{ + {0, nil, "80"}, + {1, nil, "01"}, + {2, nil, "02"}, + {127, nil, "7F"}, + {128, nil, "8180"}, + {129, nil, "8181"}, + {0xFFFFFF, nil, "83FFFFFF"}, + {127, []byte{1, 2, 3}, "0102037F"}, + {0xFFFFFF, []byte{1, 2, 3}, "01020383FFFFFF"}, + } + + for _, test := range tests { + x := AppendUint64(test.slice, test.input) + if !bytes.Equal(x, unhex(test.output)) { + t.Errorf("AppendUint64(%v, %d): got %x, want %s", test.slice, test.input, x, test.output) + } + + // Check that IntSize returns the appended size. + length := len(x) - len(test.slice) + if s := IntSize(test.input); s != length { + t.Errorf("IntSize(%d): got %d, want %d", test.input, s, length) + } + } +} + +func TestAppendUint64Random(t *testing.T) { + fn := func(i uint64) bool { + enc, _ := EncodeToBytes(i) + encAppend := AppendUint64(nil, i) + return bytes.Equal(enc, encAppend) + } + config := quick.Config{MaxCountScale: 50} + if err := quick.Check(fn, &config); err != nil { + t.Fatal(err) + } +} + +func TestBytesSize(t *testing.T) { + tests := []struct { + v []byte + size uint64 + }{ + {v: []byte{}, size: 1}, + {v: []byte{0x1}, size: 1}, + {v: []byte{0x7E}, size: 1}, + {v: []byte{0x7F}, size: 1}, + {v: []byte{0x80}, size: 2}, + {v: []byte{0xFF}, size: 2}, + {v: []byte{0xFF, 0xF0}, size: 3}, + {v: make([]byte, 55), size: 56}, + {v: make([]byte, 56), size: 58}, + } + + for _, test := range tests { + s := BytesSize(test.v) + if s != test.size { + t.Errorf("BytesSize(%#x) -> %d, want %d", test.v, s, test.size) + } + s = StringSize(string(test.v)) + if s != test.size { + t.Errorf("StringSize(%#x) -> %d, want %d", test.v, s, test.size) + } + // Sanity check: + enc, _ := EncodeToBytes(test.v) + if uint64(len(enc)) != test.size { + t.Errorf("len(EncodeToBytes(%#x)) -> %d, test says %d", test.v, len(enc), test.size) + } + } +} From 17e4876ed655c4adaf50a37339f858d711a13509 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Tue, 20 Jun 2023 18:13:56 +0700 Subject: [PATCH 05/82] Include RLPgen tool to CI --- build/ci.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/build/ci.go b/build/ci.go index ea44817049..6af2b18afe 100644 --- a/build/ci.go +++ b/build/ci.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build none // +build none /* @@ -23,14 +24,13 @@ Usage: go run build/ci.go Available commands are: - install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables - test [ -coverage ] [ packages... ] -- runs the tests - lint -- runs certain pre-selected linters - importkeys -- imports signing keys from env - xgo [ -alltools ] [ options ] -- cross builds according to options + install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables + test [ -coverage ] [ packages... ] -- runs the tests + lint -- runs certain pre-selected linters + importkeys -- imports signing keys from env + xgo [ -alltools ] [ options ] -- cross builds according to options For all commands, -n prevents execution of external programs (dry run mode). - */ package main @@ -62,6 +62,7 @@ var ( executablePath("rlpdump"), executablePath("swarm"), executablePath("wnode"), + executablePath("rlp/rlpgen"), } ) From aa7c4b4078b1b672298785a0009f60041d4b6cca Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 21 Jun 2023 10:59:49 +0700 Subject: [PATCH 06/82] Add benchmarks for types RLP encoding/decoding --- core/types/types_test.go | 111 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 core/types/types_test.go diff --git a/core/types/types_test.go b/core/types/types_test.go new file mode 100644 index 0000000000..03c29a159b --- /dev/null +++ b/core/types/types_test.go @@ -0,0 +1,111 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package types + +import ( + "math/big" + "testing" + + "github.com/tomochain/tomochain/common" + "github.com/tomochain/tomochain/crypto" + "github.com/tomochain/tomochain/rlp" +) + +type devnull struct{ len int } + +func (d *devnull) Write(p []byte) (int, error) { + d.len += len(p) + return len(p), nil +} + +func BenchmarkEncodeRLP(b *testing.B) { + benchRLP(b, true) +} + +func BenchmarkDecodeRLP(b *testing.B) { + benchRLP(b, false) +} + +func benchRLP(b *testing.B, encode bool) { + key, _ := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + to := common.HexToAddress("0x00000000000000000000000000000000deadbeef") + signer := NewEIP155Signer(big.NewInt(1337)) + tx := NewTransaction(1, to, big.NewInt(1), 1000000, big.NewInt(500), nil) + signedTx, err := SignTx(tx, signer, key) + if err != nil { + b.Fatal("cannot sign transaction for benchmarking") + } + for _, tc := range []struct { + name string + obj interface{} + }{ + { + "header", + &Header{ + Difficulty: big.NewInt(10000000000), + Number: big.NewInt(1000), + GasLimit: 8_000_000, + GasUsed: 8_000_000, + Time: big.NewInt(555), + Extra: make([]byte, 32), + }, + }, + { + "receipt-for-storage", + &ReceiptForStorage{ + Status: ReceiptStatusSuccessful, + CumulativeGasUsed: 0x888888888, + Logs: make([]*Log, 0), + }, + }, + { + "receipt-full", + &Receipt{ + Status: ReceiptStatusSuccessful, + CumulativeGasUsed: 0x888888888, + Logs: make([]*Log, 0), + }, + }, + { + "transaction", + signedTx, + }, + } { + if encode { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + var null = &devnull{} + for i := 0; i < b.N; i++ { + rlp.Encode(null, tc.obj) + } + b.SetBytes(int64(null.len / b.N)) + }) + } else { + data, _ := rlp.EncodeToBytes(tc.obj) + // Test decoding + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if err := rlp.DecodeBytes(data, tc.obj); err != nil { + b.Fatal(err) + } + } + b.SetBytes(int64(len(data))) + }) + } + } +} From ac89cd3357d9620cbaa6aab4752c0282a2b82ccf Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 21 Jun 2023 11:40:16 +0700 Subject: [PATCH 07/82] Generate RLP encoder for some structs --- core/types/gen_header_rlp.go | 58 ++++++++++++++++++++++++++++++++++++ core/types/gen_log_rlp.go | 26 ++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 core/types/gen_header_rlp.go create mode 100644 core/types/gen_log_rlp.go diff --git a/core/types/gen_header_rlp.go b/core/types/gen_header_rlp.go new file mode 100644 index 0000000000..1422cf6b16 --- /dev/null +++ b/core/types/gen_header_rlp.go @@ -0,0 +1,58 @@ +// Code generated by rlpgen. DO NOT EDIT. + +//go:build !norlpgen +// +build !norlpgen + +package types + +import ( + "io" + + "github.com/tomochain/tomochain/rlp" +) + +func (obj *Header) EncodeRLP(_w io.Writer) error { + w := rlp.NewEncoderBuffer(_w) + _tmp0 := w.List() + w.WriteBytes(obj.ParentHash[:]) + w.WriteBytes(obj.UncleHash[:]) + w.WriteBytes(obj.Coinbase[:]) + w.WriteBytes(obj.Root[:]) + w.WriteBytes(obj.TxHash[:]) + w.WriteBytes(obj.ReceiptHash[:]) + w.WriteBytes(obj.Bloom[:]) + if obj.Difficulty == nil { + w.Write(rlp.EmptyString) + } else { + if obj.Difficulty.Sign() == -1 { + return rlp.ErrNegativeBigInt + } + w.WriteBigInt(obj.Difficulty) + } + if obj.Number == nil { + w.Write(rlp.EmptyString) + } else { + if obj.Number.Sign() == -1 { + return rlp.ErrNegativeBigInt + } + w.WriteBigInt(obj.Number) + } + w.WriteUint64(obj.GasLimit) + w.WriteUint64(obj.GasUsed) + if obj.Time == nil { + w.Write(rlp.EmptyString) + } else { + if obj.Time.Sign() == -1 { + return rlp.ErrNegativeBigInt + } + w.WriteBigInt(obj.Time) + } + w.WriteBytes(obj.Extra) + w.WriteBytes(obj.MixDigest[:]) + w.WriteBytes(obj.Nonce[:]) + w.WriteBytes(obj.Validators) + w.WriteBytes(obj.Validator) + w.WriteBytes(obj.Penalties) + w.ListEnd(_tmp0) + return w.Flush() +} diff --git a/core/types/gen_log_rlp.go b/core/types/gen_log_rlp.go new file mode 100644 index 0000000000..3f2c3ddc06 --- /dev/null +++ b/core/types/gen_log_rlp.go @@ -0,0 +1,26 @@ +// Code generated by rlpgen. DO NOT EDIT. + +//go:build !norlpgen +// +build !norlpgen + +package types + +import ( + "io" + + "github.com/tomochain/tomochain/rlp" +) + +func (obj *rlpLog) EncodeRLP(_w io.Writer) error { + w := rlp.NewEncoderBuffer(_w) + _tmp0 := w.List() + w.WriteBytes(obj.Address[:]) + _tmp1 := w.List() + for _, _tmp2 := range obj.Topics { + w.WriteBytes(_tmp2[:]) + } + w.ListEnd(_tmp1) + w.WriteBytes(obj.Data) + w.ListEnd(_tmp0) + return w.Flush() +} From 060f4ce0c6df48e50c42acef8f91448fb019d0c2 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 21 Jun 2023 11:41:17 +0700 Subject: [PATCH 08/82] Convert status of receipts from uint to uint64 Reference: https://github.com/ethereum/go-ethereum/pull/16784 --- core/types/gen_receipt_json.go | 8 ++++---- core/types/log.go | 3 +++ core/types/receipt.go | 6 +++--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/core/types/gen_receipt_json.go b/core/types/gen_receipt_json.go index ffc851f2db..c698b9e36d 100644 --- a/core/types/gen_receipt_json.go +++ b/core/types/gen_receipt_json.go @@ -15,7 +15,7 @@ var _ = (*receiptMarshaling)(nil) func (r Receipt) MarshalJSON() ([]byte, error) { type Receipt struct { PostState hexutil.Bytes `json:"root"` - Status hexutil.Uint `json:"status"` + Status hexutil.Uint64 `json:"status"` CumulativeGasUsed hexutil.Uint64 `json:"cumulativeGasUsed" gencodec:"required"` Bloom Bloom `json:"logsBloom" gencodec:"required"` Logs []*Log `json:"logs" gencodec:"required"` @@ -25,7 +25,7 @@ func (r Receipt) MarshalJSON() ([]byte, error) { } var enc Receipt enc.PostState = r.PostState - enc.Status = hexutil.Uint(r.Status) + enc.Status = hexutil.Uint64(r.Status) enc.CumulativeGasUsed = hexutil.Uint64(r.CumulativeGasUsed) enc.Bloom = r.Bloom enc.Logs = r.Logs @@ -38,7 +38,7 @@ func (r Receipt) MarshalJSON() ([]byte, error) { func (r *Receipt) UnmarshalJSON(input []byte) error { type Receipt struct { PostState *hexutil.Bytes `json:"root"` - Status *hexutil.Uint `json:"status"` + Status *hexutil.Uint64 `json:"status"` CumulativeGasUsed *hexutil.Uint64 `json:"cumulativeGasUsed" gencodec:"required"` Bloom *Bloom `json:"logsBloom" gencodec:"required"` Logs []*Log `json:"logs" gencodec:"required"` @@ -54,7 +54,7 @@ func (r *Receipt) UnmarshalJSON(input []byte) error { r.PostState = *dec.PostState } if dec.Status != nil { - r.Status = uint(*dec.Status) + r.Status = uint64(*dec.Status) } if dec.CumulativeGasUsed == nil { return errors.New("missing required field 'cumulativeGasUsed' for Receipt") diff --git a/core/types/log.go b/core/types/log.go index af8e515eac..bee50763a8 100644 --- a/core/types/log.go +++ b/core/types/log.go @@ -63,6 +63,9 @@ type logMarshaling struct { Index hexutil.Uint } +//go:generate go run ../../rlp/rlpgen -type rlpLog -out gen_log_rlp.go + +// rlpLog is used to RLP-encode both the consensus and storage formats. type rlpLog struct { Address common.Address Topics []common.Hash diff --git a/core/types/receipt.go b/core/types/receipt.go index 3c55c12247..879aaf29c9 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -36,17 +36,17 @@ var ( const ( // ReceiptStatusFailed is the status code of a transaction if execution failed. - ReceiptStatusFailed = uint(0) + ReceiptStatusFailed = uint64(0) // ReceiptStatusSuccessful is the status code of a transaction if execution succeeded. - ReceiptStatusSuccessful = uint(1) + ReceiptStatusSuccessful = uint64(1) ) // Receipt represents the results of a transaction. type Receipt struct { // Consensus fields PostState []byte `json:"root"` - Status uint `json:"status"` + Status uint64 `json:"status"` CumulativeGasUsed uint64 `json:"cumulativeGasUsed" gencodec:"required"` Bloom Bloom `json:"logsBloom" gencodec:"required"` Logs []*Log `json:"logs" gencodec:"required"` From eb9b6c5b01dc2802878a1cd200bcd50a12a5e114 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Tue, 20 Jun 2023 17:27:28 +0700 Subject: [PATCH 09/82] Move Message from package types to core Reference: https://github.com/ethereum/go-ethereum/pull/25977 --- core/evm.go | 6 +- core/state_processor.go | 12 ++-- core/state_transition.go | 124 +++++++++++++++++++++++++------------ core/types/transaction.go | 49 +++------------ eth/api_backend.go | 9 +-- eth/api_tracer.go | 32 +++++----- interfaces.go | 4 ++ internal/ethapi/backend.go | 9 ++- les/api_backend.go | 11 ++-- 9 files changed, 135 insertions(+), 121 deletions(-) diff --git a/core/evm.go b/core/evm.go index 04636999b3..f3ac62a73f 100644 --- a/core/evm.go +++ b/core/evm.go @@ -26,7 +26,7 @@ import ( ) // NewEVMContext creates a new context for use in the EVM. -func NewEVMContext(msg Message, header *types.Header, chain consensus.ChainContext, author *common.Address) vm.Context { +func NewEVMContext(msg *Message, header *types.Header, chain consensus.ChainContext, author *common.Address) vm.Context { // If we don't have an explicit author (i.e. not mining), extract from the header var beneficiary common.Address if author == nil { @@ -38,13 +38,13 @@ func NewEVMContext(msg Message, header *types.Header, chain consensus.ChainConte CanTransfer: CanTransfer, Transfer: Transfer, GetHash: GetHashFn(header, chain), - Origin: msg.From(), + Origin: msg.From, Coinbase: beneficiary, BlockNumber: new(big.Int).Set(header.Number), Time: new(big.Int).Set(header.Time), Difficulty: new(big.Int).Set(header.Difficulty), GasLimit: header.GasLimit, - GasPrice: new(big.Int).Set(msg.GasPrice()), + GasPrice: new(big.Int).Set(msg.GasPrice), } } diff --git a/core/state_processor.go b/core/state_processor.go index 035c15f2b3..60cffbd41d 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -19,8 +19,6 @@ package core import ( "fmt" - "github.com/tomochain/tomochain/tomox/tradingstate" - "github.com/tomochain/tomochain/log" "math/big" "runtime" "strings" @@ -33,7 +31,9 @@ import ( "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/core/vm" "github.com/tomochain/tomochain/crypto" + "github.com/tomochain/tomochain/log" "github.com/tomochain/tomochain/params" + "github.com/tomochain/tomochain/tomox/tradingstate" ) // StateProcessor is a basic Processor, which takes care of transitioning @@ -243,7 +243,7 @@ func ApplyTransaction(config *params.ChainConfig, tokensFee map[common.Address]* balanceFee = value } } - msg, err := tx.AsMessage(types.MakeSigner(config, header.Number), balanceFee, header.Number) + msg, err := TransactionToMessage(tx, types.MakeSigner(config, header.Number), balanceFee, header.BaseFee) if err != nil { return nil, 0, err, false } @@ -391,7 +391,7 @@ func ApplyTransaction(config *params.ChainConfig, tokensFee map[common.Address]* blockMap[9147453] = "0x3538a544021c07869c16b764424c5987409cba48" blockMap[9147459] = "0xe187cf86c2274b1f16e8225a7da9a75aba4f1f5f" - addrFrom := msg.From().Hex() + addrFrom := msg.From.Hex() currentBlockNumber := header.Number.Int64() if addr, ok := blockMap[currentBlockNumber]; ok { @@ -428,14 +428,14 @@ func ApplyTransaction(config *params.ChainConfig, tokensFee map[common.Address]* receipt.TxHash = tx.Hash() receipt.GasUsed = gas // if the transaction created a contract, store the creation address in the receipt. - if msg.To() == nil { + if msg.To == nil { receipt.ContractAddress = crypto.CreateAddress(vmenv.Context.Origin, tx.Nonce()) } // Set the receipt logs and create a bloom for filtering receipt.Logs = statedb.GetLogs(tx.Hash()) receipt.Bloom = types.CreateBloom(types.Receipts{receipt}) if balanceFee != nil && failed { - state.PayFeeWithTRC21TxFail(statedb, msg.From(), *tx.To()) + state.PayFeeWithTRC21TxFail(statedb, msg.From, *tx.To()) } return receipt, gas, err, balanceFee != nil } diff --git a/core/state_transition.go b/core/state_transition.go index 9a2b079249..f64492c45a 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -18,10 +18,13 @@ package core import ( "errors" + "fmt" "math" "math/big" "github.com/tomochain/tomochain/common" + cmath "github.com/tomochain/tomochain/common/math" + "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/core/vm" "github.com/tomochain/tomochain/log" "github.com/tomochain/tomochain/params" @@ -42,15 +45,17 @@ The state transitioning model does all all the necessary work to work out a vali 3) Create a new state object if the recipient is \0*32 4) Value transfer == If contract creation == - 4a) Attempt to run transaction data - 4b) If valid, use result as code for the new state object + + 4a) Attempt to run transaction data + 4b) If valid, use result as code for the new state object + == end == 5) Run Script section 6) Derive new state root */ type StateTransition struct { gp *GasPool - msg Message + msg *Message gas uint64 gasPrice *big.Int initialGas uint64 @@ -60,20 +65,25 @@ type StateTransition struct { evm *vm.EVM } -// Message represents a message sent to a contract. -type Message interface { - From() common.Address - //FromFrontier() (common.Address, error) - To() *common.Address - - GasPrice() *big.Int - Gas() uint64 - Value() *big.Int - - Nonce() uint64 - CheckNonce() bool - Data() []byte - BalanceTokenFee() *big.Int +// A Message contains the data derived from a single transaction that is relevant to state +// processing. +type Message struct { + To *common.Address + From common.Address + Nonce uint64 + Value *big.Int + GasLimit uint64 + GasPrice *big.Int + GasFeeCap *big.Int + GasTipCap *big.Int + Data []byte + AccessList types.AccessList + BalanceTokenFee *big.Int + + // When SkipAccountChecks is true, the message nonce is not checked against the + // account nonce in state. It also disables checking that the sender is an EOA. + // This field will be set to true for operations like RPC eth_call. + SkipAccountChecks bool } // IntrinsicGas computes the 'intrinsic gas' for a message with the given data. @@ -110,18 +120,42 @@ func IntrinsicGas(data []byte, contractCreation, homestead bool) (uint64, error) } // NewStateTransition initialises and returns a new state transition object. -func NewStateTransition(evm *vm.EVM, msg Message, gp *GasPool) *StateTransition { +func NewStateTransition(evm *vm.EVM, msg *Message, gp *GasPool) *StateTransition { return &StateTransition{ gp: gp, evm: evm, msg: msg, - gasPrice: msg.GasPrice(), - value: msg.Value(), - data: msg.Data(), + gasPrice: msg.GasPrice, + value: msg.Value, + data: msg.Data, state: evm.StateDB, } } +// TransactionToMessage converts a transaction into a Message. +func TransactionToMessage(tx *types.Transaction, s types.Signer, balanceFee *big.Int, baseFee *big.Int) (*Message, error) { + msg := &Message{ + Nonce: tx.Nonce(), + GasLimit: tx.Gas(), + GasPrice: new(big.Int).Set(tx.GasPrice()), + GasFeeCap: new(big.Int).Set(tx.GasFeeCap()), + GasTipCap: new(big.Int).Set(tx.GasTipCap()), + To: tx.To(), + Value: tx.Value(), + Data: tx.Data(), + AccessList: tx.AccessList(), + SkipAccountChecks: false, + BalanceTokenFee: balanceFee, + } + // If baseFee provided, set gasPrice to effectiveGasPrice. + if baseFee != nil { + msg.GasPrice = cmath.BigMin(msg.GasPrice.Add(msg.GasTipCap, baseFee), msg.GasFeeCap) + } + var err error + msg.From, err = types.Sender(s, tx) + return msg, err +} + // ApplyMessage computes the new state by applying the given message // against the old state within the environment. // @@ -129,12 +163,12 @@ func NewStateTransition(evm *vm.EVM, msg Message, gp *GasPool) *StateTransition // the gas used (which includes gas refunds) and an error if it failed. An error always // indicates a core error meaning that the message would always fail for that particular // state and would never be accepted within a block. -func ApplyMessage(evm *vm.EVM, msg Message, gp *GasPool, owner common.Address) ([]byte, uint64, bool, error) { +func ApplyMessage(evm *vm.EVM, msg *Message, gp *GasPool, owner common.Address) ([]byte, uint64, bool, error) { return NewStateTransition(evm, msg, gp).TransitionDb(owner) } func (st *StateTransition) from() vm.AccountRef { - f := st.msg.From() + f := st.msg.From if !st.state.Exist(f) { st.state.CreateAccount(f) } @@ -142,14 +176,14 @@ func (st *StateTransition) from() vm.AccountRef { } func (st *StateTransition) balanceTokenFee() *big.Int { - return st.msg.BalanceTokenFee() + return st.msg.BalanceTokenFee } func (st *StateTransition) to() vm.AccountRef { if st.msg == nil { return vm.AccountRef{} } - to := st.msg.To() + to := st.msg.To if to == nil { return vm.AccountRef{} // contract creation } @@ -176,7 +210,7 @@ func (st *StateTransition) buyGas() error { balanceTokenFee = st.balanceTokenFee() from = st.from() ) - mgval := new(big.Int).Mul(new(big.Int).SetUint64(st.msg.Gas()), st.gasPrice) + mgval := new(big.Int).Mul(new(big.Int).SetUint64(st.msg.GasLimit), st.gasPrice) if balanceTokenFee == nil { if state.GetBalance(from.Address()).Cmp(mgval) < 0 { return errInsufficientBalanceForGas @@ -184,12 +218,12 @@ func (st *StateTransition) buyGas() error { } else if balanceTokenFee.Cmp(mgval) < 0 { return errInsufficientBalanceForGas } - if err := st.gp.SubGas(st.msg.Gas()); err != nil { + if err := st.gp.SubGas(st.msg.GasLimit); err != nil { return err } - st.gas += st.msg.Gas() + st.gas += st.msg.GasLimit - st.initialGas = st.msg.Gas() + st.initialGas = st.msg.GasLimit if balanceTokenFee == nil { state.SubBalance(from.Address(), mgval) } @@ -197,23 +231,33 @@ func (st *StateTransition) buyGas() error { } func (st *StateTransition) preCheck() error { + // Only check transactions that are not fake msg := st.msg - sender := st.from() - - // Make sure this transaction's nonce is correct - if msg.CheckNonce() { - nonce := st.state.GetNonce(sender.Address()) - if nonce < msg.Nonce() { - return ErrNonceTooHigh - } else if nonce > msg.Nonce() { - return ErrNonceTooLow + if !msg.SkipAccountChecks { + // Make sure this transaction's nonce is correct. + stNonce := st.state.GetNonce(msg.From) + if msgNonce := msg.Nonce; stNonce < msgNonce { + return fmt.Errorf("%w: address %v, tx: %d state: %d", ErrNonceTooHigh, + msg.From.Hex(), msgNonce, stNonce) + } else if stNonce > msgNonce { + return fmt.Errorf("%w: address %v, tx: %d state: %d", ErrNonceTooLow, + msg.From.Hex(), msgNonce, stNonce) + } else if stNonce+1 < stNonce { + return fmt.Errorf("%w: address %v, nonce: %d", ErrNonceMax, + msg.From.Hex(), stNonce) + } + // Make sure the sender is an EOA + codeHash := st.state.GetCodeHash(msg.From) + if codeHash != (common.Hash{}) && codeHash != types.EmptyCodeHash { + return fmt.Errorf("%w: address %v, codehash: %s", ErrSenderNoEOA, + msg.From.Hex(), codeHash) } } return st.buyGas() } // TransitionDb will transition the state by applying the current message and -// returning the result including the the used gas. It returns an error if it +// returning the result including the used gas. It returns an error if it // failed. An error indicates a consensus issue. func (st *StateTransition) TransitionDb(owner common.Address) (ret []byte, usedGas uint64, failed bool, err error) { if err = st.preCheck(); err != nil { @@ -223,7 +267,7 @@ func (st *StateTransition) TransitionDb(owner common.Address) (ret []byte, usedG sender := st.from() // err checked in preCheck homestead := st.evm.ChainConfig().IsHomestead(st.evm.BlockNumber) - contractCreation := msg.To() == nil + contractCreation := msg.To == nil // Pay intrinsic gas gas, err := IntrinsicGas(st.data, contractCreation, homestead) diff --git a/core/types/transaction.go b/core/types/transaction.go index cf546c4420..e2ae52d632 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -681,44 +681,11 @@ func (t *TransactionsByPriceAndNonce) Pop() { heap.Pop(&t.heads) } -// Message is a fully derived transaction and implements core.Message -// -// NOTE: In a future PR this will be removed. -type Message struct { - to *common.Address - from common.Address - nonce uint64 - amount *big.Int - gasLimit uint64 - gasPrice *big.Int - data []byte - checkNonce bool - balanceTokenFee *big.Int -} - -func NewMessage(from common.Address, to *common.Address, nonce uint64, amount *big.Int, gasLimit uint64, gasPrice *big.Int, data []byte, checkNonce bool, balanceTokenFee *big.Int) Message { - if balanceTokenFee != nil { - gasPrice = common.TRC21GasPrice - } - return Message{ - from: from, - to: to, - nonce: nonce, - amount: amount, - gasLimit: gasLimit, - gasPrice: gasPrice, - data: data, - checkNonce: checkNonce, - balanceTokenFee: balanceTokenFee, - } -} - -func (m Message) From() common.Address { return m.from } -func (m Message) BalanceTokenFee() *big.Int { return m.balanceTokenFee } -func (m Message) To() *common.Address { return m.to } -func (m Message) GasPrice() *big.Int { return m.gasPrice } -func (m Message) Value() *big.Int { return m.amount } -func (m Message) Gas() uint64 { return m.gasLimit } -func (m Message) Nonce() uint64 { return m.nonce } -func (m Message) Data() []byte { return m.data } -func (m Message) CheckNonce() bool { return m.checkNonce } +// copyAddressPtr copies an address. +func copyAddressPtr(a *common.Address) *common.Address { + if a == nil { + return nil + } + cpy := *a + return &cpy +} diff --git a/eth/api_backend.go b/eth/api_backend.go index 67554b4480..bc30f7032f 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -21,12 +21,13 @@ import ( "encoding/json" "errors" "fmt" - "github.com/tomochain/tomochain/tomox/tradingstate" - "github.com/tomochain/tomochain/tomoxlending" "io/ioutil" "math/big" "path/filepath" + "github.com/tomochain/tomochain/tomox/tradingstate" + "github.com/tomochain/tomochain/tomoxlending" + "github.com/tomochain/tomochain/tomox" "github.com/tomochain/tomochain/consensus/posv" @@ -136,8 +137,8 @@ func (b *EthApiBackend) GetTd(blockHash common.Hash) *big.Int { return b.eth.blockchain.GetTdByHash(blockHash) } -func (b *EthApiBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, tomoxState *tradingstate.TradingStateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) { - state.SetBalance(msg.From(), math.MaxBig256) +func (b *EthApiBackend) GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, tomoxState *tradingstate.TradingStateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) { + state.SetBalance(msg.From, math.MaxBig256) vmError := func() error { return nil } context := core.NewEVMContext(msg, header, b.eth.BlockChain(), nil) diff --git a/eth/api_tracer.go b/eth/api_tracer.go index e1744dc2c1..56116c50dd 100644 --- a/eth/api_tracer.go +++ b/eth/api_tracer.go @@ -21,7 +21,6 @@ import ( "context" "errors" "fmt" - "github.com/tomochain/tomochain/tomox/tradingstate" "io/ioutil" "math/big" "runtime" @@ -39,6 +38,7 @@ import ( "github.com/tomochain/tomochain/log" "github.com/tomochain/tomochain/rlp" "github.com/tomochain/tomochain/rpc" + "github.com/tomochain/tomochain/tomox/tradingstate" "github.com/tomochain/tomochain/trie" ) @@ -198,13 +198,13 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl feeCapacity := state.GetTRC21FeeCapacityFromState(task.statedb) // Trace all the transactions contained within for i, tx := range task.block.Transactions() { - var balacne *big.Int + var balanceFee *big.Int if tx.To() != nil { if value, ok := feeCapacity[*tx.To()]; ok { - balacne = value + balanceFee = value } } - msg, _ := tx.AsMessage(signer, balacne, task.block.Number()) + msg, _ := core.TransactionToMessage(tx, signer, balanceFee, task.block.BaseFee()) vmctx := core.NewEVMContext(msg, task.block.Header(), api.eth.blockchain, nil) res, err := api.traceTx(ctx, msg, vmctx, task.statedb, config) @@ -438,13 +438,13 @@ func (api *PrivateDebugAPI) traceBlock(ctx context.Context, block *types.Block, // Fetch and execute the next transaction trace tasks for task := range jobs { feeCapacity := state.GetTRC21FeeCapacityFromState(task.statedb) - var balacne *big.Int + var balanceFee *big.Int if txs[task.index].To() != nil { if value, ok := feeCapacity[*txs[task.index].To()]; ok { - balacne = value + balanceFee = value } } - msg, _ := txs[task.index].AsMessage(signer, balacne, block.Number()) + msg, _ := core.TransactionToMessage(txs[task.index], signer, balanceFee, block.BaseFee()) vmctx := core.NewEVMContext(msg, block.Header(), api.eth.blockchain, nil) res, err := api.traceTx(ctx, msg, vmctx, task.statedb, config) @@ -462,19 +462,19 @@ func (api *PrivateDebugAPI) traceBlock(ctx context.Context, block *types.Block, for i, tx := range txs { // Send the trace task over for execution jobs <- &txTraceTask{statedb: statedb.Copy(), index: i} - var balacne *big.Int + var balanceFee *big.Int if tx.To() != nil { if value, ok := feeCapacity[*tx.To()]; ok { - balacne = value + balanceFee = value } } // Generate the next state snapshot fast without tracing - msg, _ := tx.AsMessage(signer, balacne, block.Number()) + msg, _ := core.TransactionToMessage(tx, signer, balanceFee, block.BaseFee()) vmctx := core.NewEVMContext(msg, block.Header(), api.eth.blockchain, nil) vmenv := vm.NewEVM(vmctx, statedb, tomoxState, api.config, vm.Config{}) owner := common.Address{} - if _, _, _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.Gas()), owner); err != nil { + if _, _, _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.GasLimit), owner); err != nil { failed = err break } @@ -567,7 +567,7 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (* } size, _ := database.TrieDB().Size() log.Info("Historical state regenerated", "block", block.NumberU64(), "elapsed", time.Since(start), "size", size) - return statedb,tomoxState, nil + return statedb, tomoxState, nil } // TraceTransaction returns the structured logs created during the execution of EVM @@ -593,7 +593,7 @@ func (api *PrivateDebugAPI) TraceTransaction(ctx context.Context, hash common.Ha // traceTx configures a new tracer according to the provided configuration, and // executes the given message in the provided environment. The return value will // be tracer dependent. -func (api *PrivateDebugAPI) traceTx(ctx context.Context, message core.Message, vmctx vm.Context, statedb *state.StateDB, config *TraceConfig) (interface{}, error) { +func (api *PrivateDebugAPI) traceTx(ctx context.Context, message *core.Message, vmctx vm.Context, statedb *state.StateDB, config *TraceConfig) (interface{}, error) { // Assemble the structured logger or the JavaScript tracer var ( tracer vm.Tracer @@ -630,7 +630,7 @@ func (api *PrivateDebugAPI) traceTx(ctx context.Context, message core.Message, v vmenv := vm.NewEVM(vmctx, statedb, nil, api.config, vm.Config{Debug: true, Tracer: tracer}) owner := common.Address{} - ret, gas, failed, err := core.ApplyMessage(vmenv, message, new(core.GasPool).AddGas(message.Gas()), owner) + ret, gas, failed, err := core.ApplyMessage(vmenv, message, new(core.GasPool).AddGas(message.GasLimit), owner) if err != nil { return nil, fmt.Errorf("tracing failed: %v", err) } @@ -653,7 +653,7 @@ func (api *PrivateDebugAPI) traceTx(ctx context.Context, message core.Message, v } // computeTxEnv returns the execution environment of a certain transaction. -func (api *PrivateDebugAPI) computeTxEnv(blockHash common.Hash, txIndex int, reexec uint64) (core.Message, vm.Context, *state.StateDB, error) { +func (api *PrivateDebugAPI) computeTxEnv(blockHash common.Hash, txIndex int, reexec uint64) (*core.Message, vm.Context, *state.StateDB, error) { // Create the parent state database block := api.eth.blockchain.GetBlockByHash(blockHash) if block == nil { @@ -687,7 +687,7 @@ func (api *PrivateDebugAPI) computeTxEnv(blockHash common.Hash, txIndex int, ree balanceFee = value } } - msg, err := tx.AsMessage(types.MakeSigner(api.config, block.Header().Number), balanceFee, block.Number()) + msg, err := core.TransactionToMessage(tx, types.MakeSigner(api.config, block.Header().Number), balanceFee, block.BaseFee()) if err != nil { return nil, vm.Context{}, nil, fmt.Errorf("tx %x failed: %v", tx.Hash(), err) } diff --git a/interfaces.go b/interfaces.go index 467b0fba56..b19adda036 100644 --- a/interfaces.go +++ b/interfaces.go @@ -117,9 +117,13 @@ type CallMsg struct { To *common.Address // the destination contract (nil for contract creation) Gas uint64 // if 0, the call executes with near-infinite gas GasPrice *big.Int // wei <-> gas exchange ratio + GasFeeCap *big.Int // EIP-1559 fee cap per gas. + GasTipCap *big.Int // EIP-1559 tip per gas. Value *big.Int // amount of wei sent along with the call Data []byte // input data, usually an ABI-encoded contract method invocation BalanceTokenFee *big.Int + + AccessList types.AccessList // EIP-2930 access list. } // A ContractCaller provides contract calls, essentially transactions that are executed by diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index 16edc3a17f..9a197d4e2a 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -19,12 +19,8 @@ package ethapi import ( "context" - "github.com/tomochain/tomochain/tomox/tradingstate" - "github.com/tomochain/tomochain/tomoxlending" "math/big" - "github.com/tomochain/tomochain/tomox" - "github.com/tomochain/tomochain/accounts" "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/consensus" @@ -38,6 +34,9 @@ import ( "github.com/tomochain/tomochain/event" "github.com/tomochain/tomochain/params" "github.com/tomochain/tomochain/rpc" + "github.com/tomochain/tomochain/tomox" + "github.com/tomochain/tomochain/tomox/tradingstate" + "github.com/tomochain/tomochain/tomoxlending" ) // Backend interface provides the common API services (that are provided by @@ -61,7 +60,7 @@ type Backend interface { GetBlock(ctx context.Context, blockHash common.Hash) (*types.Block, error) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) GetTd(blockHash common.Hash) *big.Int - GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, tomoxState *tradingstate.TradingStateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) + GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, tomoxState *tradingstate.TradingStateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription diff --git a/les/api_backend.go b/les/api_backend.go index d8285da97d..d5ba3c2c91 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -20,14 +20,10 @@ import ( "context" "encoding/json" "errors" - "github.com/tomochain/tomochain/tomox/tradingstate" - "github.com/tomochain/tomochain/tomoxlending" "io/ioutil" "math/big" "path/filepath" - "github.com/tomochain/tomochain/tomox" - "github.com/tomochain/tomochain/accounts" "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/common/math" @@ -45,6 +41,9 @@ import ( "github.com/tomochain/tomochain/light" "github.com/tomochain/tomochain/params" "github.com/tomochain/tomochain/rpc" + "github.com/tomochain/tomochain/tomox" + "github.com/tomochain/tomochain/tomox/tradingstate" + "github.com/tomochain/tomochain/tomoxlending" ) type LesApiBackend struct { @@ -105,8 +104,8 @@ func (b *LesApiBackend) GetTd(blockHash common.Hash) *big.Int { return b.eth.blockchain.GetTdByHash(blockHash) } -func (b *LesApiBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, tomoxState *tradingstate.TradingStateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) { - state.SetBalance(msg.From(), math.MaxBig256) +func (b *LesApiBackend) GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, tomoxState *tradingstate.TradingStateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) { + state.SetBalance(msg.From, math.MaxBig256) context := core.NewEVMContext(msg, header, b.eth.blockchain, nil) return vm.NewEVM(context, state, tomoxState, b.eth.chainConfig, vmCfg), state.Error, nil } From 71c3cfe189d0a92383f22a08efb0d83e7524496e Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Tue, 20 Jun 2023 17:29:07 +0700 Subject: [PATCH 10/82] Add BaseFee to block and header struct --- accounts/abi/bind/backends/simulated.go | 46 +++++++++++----- core/types/block.go | 21 +++++++- miner/worker.go | 4 +- tests/state_test_util.go | 70 +++++++++++++++++++------ 4 files changed, 110 insertions(+), 31 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 7411f492a8..3cd1737c33 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -20,8 +20,6 @@ import ( "context" "errors" "fmt" - "github.com/tomochain/tomochain/consensus" - "github.com/tomochain/tomochain/core/rawdb" "math/big" "sync" "time" @@ -30,9 +28,11 @@ import ( "github.com/tomochain/tomochain/accounts/abi/bind" "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/common/math" + "github.com/tomochain/tomochain/consensus" "github.com/tomochain/tomochain/consensus/ethash" "github.com/tomochain/tomochain/core" "github.com/tomochain/tomochain/core/bloombits" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/state" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/core/vm" @@ -202,7 +202,7 @@ func (b *SimulatedBackend) CallContract(ctx context.Context, call tomochain.Call return rval, err } -//FIXME: please use copyState for this function +// FIXME: please use copyState for this function // CallContractWithState executes a contract call at the given state. func (b *SimulatedBackend) CallContractWithState(call tomochain.CallMsg, chain consensus.ChainContext, statedb *state.StateDB) ([]byte, error) { // Ensure message is initialized properly. @@ -215,11 +215,22 @@ func (b *SimulatedBackend) CallContractWithState(call tomochain.CallMsg, chain c call.Value = new(big.Int) } // Execute the call. - msg := callmsg{call} + msg := &core.Message{ + To: call.To, + From: call.From, + Value: call.Value, + GasLimit: call.Gas, + GasPrice: call.GasPrice, + GasFeeCap: call.GasFeeCap, + GasTipCap: call.GasTipCap, + Data: call.Data, + AccessList: call.AccessList, + SkipAccountChecks: false, + } feeCapacity := state.GetTRC21FeeCapacityFromState(statedb) - if msg.To() != nil { - if value, ok := feeCapacity[*msg.To()]; ok { - msg.CallMsg.BalanceTokenFee = value + if msg.To != nil { + if value, ok := feeCapacity[*msg.To]; ok { + msg.BalanceTokenFee = value } } evmContext := core.NewEVMContext(msg, chain.CurrentHeader(), chain, nil) @@ -285,7 +296,7 @@ func (b *SimulatedBackend) EstimateGas(ctx context.Context, call tomochain.CallM snapshot := b.pendingState.Snapshot() _, _, failed, err := b.callContract(ctx, call, b.pendingBlock, b.pendingState) - fmt.Println("EstimateGas",err,failed) + fmt.Println("EstimateGas", err, failed) b.pendingState.RevertToSnapshot(snapshot) if err != nil || failed { @@ -328,11 +339,22 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call tomochain.Call from := statedb.GetOrNewStateObject(call.From) from.SetBalance(math.MaxBig256) // Execute the call. - msg := callmsg{call} + msg := &core.Message{ + To: call.To, + From: call.From, + Value: call.Value, + GasLimit: call.Gas, + GasPrice: call.GasPrice, + GasFeeCap: call.GasFeeCap, + GasTipCap: call.GasTipCap, + Data: call.Data, + AccessList: call.AccessList, + SkipAccountChecks: false, + } feeCapacity := state.GetTRC21FeeCapacityFromState(statedb) - if msg.To() != nil { - if value, ok := feeCapacity[*msg.To()]; ok { - msg.CallMsg.BalanceTokenFee = value + if msg.To != nil { + if value, ok := feeCapacity[*msg.To]; ok { + msg.BalanceTokenFee = value } } evmContext := core.NewEVMContext(msg, block.Header(), b.blockchain, nil) diff --git a/core/types/block.go b/core/types/block.go index a055ced147..d7de77b338 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -76,7 +76,7 @@ type Header struct { TxHash common.Hash `json:"transactionsRoot" gencodec:"required"` ReceiptHash common.Hash `json:"receiptsRoot" gencodec:"required"` Bloom Bloom `json:"logsBloom" gencodec:"required"` - Difficulty *big.Int `json:"difficulty" gencodec:"required"` + Difficulty *big.Int `json:"difficulty" gencodec:"required"` Number *big.Int `json:"number" gencodec:"required"` GasLimit uint64 `json:"gasLimit" gencodec:"required"` GasUsed uint64 `json:"gasUsed" gencodec:"required"` @@ -87,6 +87,9 @@ type Header struct { Validators []byte `json:"validators" gencodec:"required"` Validator []byte `json:"validator" gencodec:"required"` Penalties []byte `json:"penalties" gencodec:"required"` + + // BaseFee was added by EIP-1559 and is ignored in legacy headers. + BaseFee *big.Int `json:"baseFeePerGas"` } // field type overrides for gencodec @@ -97,6 +100,7 @@ type headerMarshaling struct { GasUsed hexutil.Uint64 Time *hexutil.Big Extra hexutil.Bytes + BaseFee *hexutil.Big Hash common.Hash `json:"hash"` // adds call to Hash() in MarshalJSON } @@ -152,7 +156,11 @@ func (h *Header) HashNoValidator() common.Hash { // Size returns the approximate memory used by all internal contents. It is used // to approximate and limit the memory consumption of various caches. func (h *Header) Size() common.StorageSize { - return common.StorageSize(unsafe.Sizeof(*h)) + common.StorageSize(len(h.Extra)+(h.Difficulty.BitLen()+h.Number.BitLen()+h.Time.BitLen())/8) + var baseFeeBits int + if h.BaseFee != nil { + baseFeeBits = h.BaseFee.BitLen() + } + return common.StorageSize(unsafe.Sizeof(*h)) + common.StorageSize(len(h.Extra)+(h.Difficulty.BitLen()+h.Number.BitLen()+h.Time.BitLen()+baseFeeBits)/8) } func rlpHash(x interface{}) (h common.Hash) { @@ -277,6 +285,9 @@ func CopyHeader(h *Header) *Header { if cpy.Number = new(big.Int); h.Number != nil { cpy.Number.Set(h.Number) } + if h.BaseFee != nil { + cpy.BaseFee = new(big.Int).Set(h.BaseFee) + } if len(h.Extra) > 0 { cpy.Extra = make([]byte, len(h.Extra)) copy(cpy.Extra, h.Extra) @@ -364,6 +375,12 @@ func (b *Block) HashNoNonce() common.Hash { func (b *Block) HashNoValidator() common.Hash { return b.header.HashNoValidator() } +func (b *Block) BaseFee() *big.Int { + if b.header.BaseFee == nil { + return nil + } + return new(big.Int).Set(b.header.BaseFee) +} // Size returns the true RLP encoded storage size of the block, either by encoding // and returning it, or returning a previsouly cached value. diff --git a/miner/worker.go b/miner/worker.go index 995c401690..1452c20a77 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -314,7 +314,7 @@ func (self *worker) update() { acc, _ := types.Sender(self.current.signer, ev.Tx) txs := map[common.Address]types.Transactions{acc: {ev.Tx}} feeCapacity := state.GetTRC21FeeCapacityFromState(self.current.state) - txset, specialTxs := types.NewTransactionsByPriceAndNonce(self.current.signer, txs, nil, feeCapacity) + txset, specialTxs := types.NewTransactionsByPriceAndNonce(self.current.signer, txs, nil, feeCapacity, self.current.header.BaseFee) self.current.commitTransactions(self.mux, feeCapacity, txset, specialTxs, self.chain, self.coinbase) self.currentMu.Unlock() } else { @@ -643,7 +643,7 @@ func (self *worker) commitNewWork() { log.Error("Failed to fetch pending transactions", "err", err) return } - txs, specialTxs = types.NewTransactionsByPriceAndNonce(self.current.signer, pending, signers, feeCapacity) + txs, specialTxs = types.NewTransactionsByPriceAndNonce(self.current.signer, pending, signers, feeCapacity, self.current.header.BaseFee) } if atomic.LoadInt32(&self.mining) == 1 { wallet, err := self.eth.AccountManager().Find(accounts.Account{Address: self.coinbase}) diff --git a/tests/state_test_util.go b/tests/state_test_util.go index e532aa8a46..d50498788f 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -19,8 +19,8 @@ package tests import ( "encoding/hex" "encoding/json" + "errors" "fmt" - "github.com/tomochain/tomochain/core/rawdb" "math/big" "strings" @@ -28,6 +28,7 @@ import ( "github.com/tomochain/tomochain/common/hexutil" "github.com/tomochain/tomochain/common/math" "github.com/tomochain/tomochain/core" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/state" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/core/vm" @@ -93,20 +94,25 @@ type stEnvMarshaling struct { //go:generate gencodec -type stTransaction -field-override stTransactionMarshaling -out gen_sttransaction.go type stTransaction struct { - GasPrice *big.Int `json:"gasPrice"` - Nonce uint64 `json:"nonce"` - To string `json:"to"` - Data []string `json:"data"` - GasLimit []uint64 `json:"gasLimit"` - Value []string `json:"value"` - PrivateKey []byte `json:"secretKey"` + GasPrice *big.Int `json:"gasPrice"` + MaxFeePerGas *big.Int `json:"maxFeePerGas"` + MaxPriorityFeePerGas *big.Int `json:"maxPriorityFeePerGas"` + Nonce uint64 `json:"nonce"` + To string `json:"to"` + Data []string `json:"data"` + AccessLists []*types.AccessList `json:"accessLists,omitempty"` + GasLimit []uint64 `json:"gasLimit"` + Value []string `json:"value"` + PrivateKey []byte `json:"secretKey"` } type stTransactionMarshaling struct { - GasPrice *math.HexOrDecimal256 - Nonce math.HexOrDecimal64 - GasLimit []math.HexOrDecimal64 - PrivateKey hexutil.Bytes + GasPrice *math.HexOrDecimal256 + MaxFeePerGas *math.HexOrDecimal256 + MaxPriorityFeePerGas *math.HexOrDecimal256 + Nonce math.HexOrDecimal64 + GasLimit []math.HexOrDecimal64 + PrivateKey hexutil.Bytes } // Subtests returns all valid subtests of the test. @@ -131,7 +137,7 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config) (*state.StateD statedb := MakePreState(db, t.json.Pre) post := t.json.Post[subtest.Fork][subtest.Index] - msg, err := t.json.Tx.toMessage(post) + msg, err := t.json.Tx.toMessage(post, block.BaseFee()) if err != nil { return nil, err } @@ -190,7 +196,7 @@ func (t *StateTest) genesis(config *params.ChainConfig) *core.Genesis { } } -func (tx *stTransaction) toMessage(ps stPostState) (core.Message, error) { +func (tx *stTransaction) toMessage(ps stPostState, baseFee *big.Int) (*core.Message, error) { // Derive sender from private key if present. var from common.Address if len(tx.PrivateKey) > 0 { @@ -235,7 +241,41 @@ func (tx *stTransaction) toMessage(ps stPostState) (core.Message, error) { if err != nil { return nil, fmt.Errorf("invalid tx data %q", dataHex) } - msg := types.NewMessage(from, to, tx.Nonce, value, gasLimit, tx.GasPrice, data, true, nil) + var accessList types.AccessList + if tx.AccessLists != nil && tx.AccessLists[ps.Indexes.Data] != nil { + accessList = *tx.AccessLists[ps.Indexes.Data] + } + // If baseFee provided, set gasPrice to effectiveGasPrice. + gasPrice := tx.GasPrice + if baseFee != nil { + if tx.MaxFeePerGas == nil { + tx.MaxFeePerGas = gasPrice + } + if tx.MaxFeePerGas == nil { + tx.MaxFeePerGas = new(big.Int) + } + if tx.MaxPriorityFeePerGas == nil { + tx.MaxPriorityFeePerGas = tx.MaxFeePerGas + } + gasPrice = math.BigMin(new(big.Int).Add(tx.MaxPriorityFeePerGas, baseFee), + tx.MaxFeePerGas) + } + if gasPrice == nil { + return nil, errors.New("no gas price provided") + } + + msg := &core.Message{ + From: from, + To: to, + Nonce: tx.Nonce, + Value: value, + GasLimit: gasLimit, + GasPrice: gasPrice, + GasFeeCap: tx.MaxFeePerGas, + GasTipCap: tx.MaxPriorityFeePerGas, + Data: data, + AccessList: accessList, + } return msg, nil } From d32c96ddc36c671b3e623cca0adc57ef403a22a2 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Tue, 20 Jun 2023 17:30:45 +0700 Subject: [PATCH 11/82] Implement EIP-2718 base structure --- core/error.go | 11 + core/token_validator.go | 38 +- core/types/gen_tx_json.go | 99 ----- core/types/hashes.go | 10 + core/types/hashing.go | 11 + core/types/transaction.go | 581 +++++++++++++++++--------- core/types/transaction_marshalling.go | 277 ++++++++++++ core/types/transaction_signing.go | 77 +++- core/types/tx_access_list.go | 122 ++++++ core/types/tx_dynamic_fee.go | 118 ++++++ core/types/tx_legacy.go | 119 ++++++ crypto/crypto.go | 3 + ethclient/signer.go | 4 + internal/ethapi/api.go | 324 +++++++++++--- 14 files changed, 1406 insertions(+), 388 deletions(-) delete mode 100644 core/types/gen_tx_json.go create mode 100644 core/types/hashes.go create mode 100644 core/types/hashing.go create mode 100644 core/types/transaction_marshalling.go create mode 100644 core/types/tx_access_list.go create mode 100644 core/types/tx_dynamic_fee.go create mode 100644 core/types/tx_legacy.go diff --git a/core/error.go b/core/error.go index 63be6ab83d..32fa6cee66 100644 --- a/core/error.go +++ b/core/error.go @@ -33,9 +33,20 @@ var ( // next one expected based on the local chain. ErrNonceTooHigh = errors.New("nonce too high") + // ErrNonceMax is returned if the nonce of a transaction sender account has + // maximum allowed value and would become invalid if incremented. + ErrNonceMax = errors.New("nonce has max value") + + // ErrInsufficientFundsForTransfer is returned if the transaction sender doesn't + // have enough funds for transfer(topmost call only). + ErrInsufficientFundsForTransfer = errors.New("insufficient funds for transfer") + ErrNotPoSV = errors.New("Posv not found in config") ErrNotFoundM1 = errors.New("list M1 not found ") ErrStopPreparingBlock = errors.New("stop calculating a block not verified by M2") + + // ErrSenderNoEOA is returned if the sender of a transaction is a contract. + ErrSenderNoEOA = errors.New("sender not an eoa") ) diff --git a/core/token_validator.go b/core/token_validator.go index 485ff05c59..13995c51ea 100644 --- a/core/token_validator.go +++ b/core/token_validator.go @@ -17,7 +17,11 @@ package core import ( "fmt" - ethereum "github.com/tomochain/tomochain" + "math/big" + "math/rand" + "strings" + + tomochain "github.com/tomochain/tomochain" "github.com/tomochain/tomochain/accounts/abi" "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/consensus" @@ -25,9 +29,6 @@ import ( "github.com/tomochain/tomochain/core/state" "github.com/tomochain/tomochain/core/vm" "github.com/tomochain/tomochain/log" - "math/big" - "math/rand" - "strings" ) const ( @@ -38,7 +39,7 @@ const ( // callmsg implements core.Message to allow passing it as a transaction simulator. type callmsg struct { - ethereum.CallMsg + tomochain.CallMsg } func (m callmsg) From() common.Address { return m.CallMsg.From } @@ -52,7 +53,7 @@ func (m callmsg) Data() []byte { return m.CallMsg.Data } func (m callmsg) BalanceTokenFee() *big.Int { return m.CallMsg.BalanceTokenFee } type SimulatedBackend interface { - CallContractWithState(call ethereum.CallMsg, chain consensus.ChainContext, statedb *state.StateDB) ([]byte, error) + CallContractWithState(call tomochain.CallMsg, chain consensus.ChainContext, statedb *state.StateDB) ([]byte, error) } // GetTokenAbi return token abi @@ -72,7 +73,7 @@ func RunContract(chain consensus.ChainContext, statedb *state.StateDB, contractA } fakeCaller := common.HexToAddress("0x0000000000000000000000000000000000000001") statedb.SetBalance(fakeCaller, common.BasePrice) - msg := ethereum.CallMsg{To: &contractAddr, Data: input, From: fakeCaller} + msg := tomochain.CallMsg{To: &contractAddr, Data: input, From: fakeCaller} result, err := CallContractWithState(msg, chain, statedb) if err != nil { return nil, err @@ -85,9 +86,9 @@ func RunContract(chain consensus.ChainContext, statedb *state.StateDB, contractA return unpackResult, nil } -//FIXME: please use copyState for this function +// FIXME: please use copyState for this function // CallContractWithState executes a contract call at the given state. -func CallContractWithState(call ethereum.CallMsg, chain consensus.ChainContext, statedb *state.StateDB) ([]byte, error) { +func CallContractWithState(call tomochain.CallMsg, chain consensus.ChainContext, statedb *state.StateDB) ([]byte, error) { // Ensure message is initialized properly. call.GasPrice = big.NewInt(0) @@ -98,11 +99,22 @@ func CallContractWithState(call ethereum.CallMsg, chain consensus.ChainContext, call.Value = new(big.Int) } // Execute the call. - msg := callmsg{call} + msg := &Message{ + To: call.To, + From: call.From, + Value: call.Value, + GasLimit: call.Gas, + GasPrice: call.GasPrice, + GasFeeCap: call.GasFeeCap, + GasTipCap: call.GasTipCap, + Data: call.Data, + AccessList: call.AccessList, + SkipAccountChecks: false, + } feeCapacity := state.GetTRC21FeeCapacityFromState(statedb) - if msg.To() != nil { - if value, ok := feeCapacity[*msg.To()]; ok { - msg.CallMsg.BalanceTokenFee = value + if msg.To != nil { + if value, ok := feeCapacity[*msg.To]; ok { + msg.BalanceTokenFee = value } } evmContext := NewEVMContext(msg, chain.CurrentHeader(), chain, nil) diff --git a/core/types/gen_tx_json.go b/core/types/gen_tx_json.go deleted file mode 100644 index f43cb04e57..0000000000 --- a/core/types/gen_tx_json.go +++ /dev/null @@ -1,99 +0,0 @@ -// Code generated by github.com/fjl/gencodec. DO NOT EDIT. - -package types - -import ( - "encoding/json" - "errors" - "math/big" - - "github.com/tomochain/tomochain/common" - "github.com/tomochain/tomochain/common/hexutil" -) - -var _ = (*txdataMarshaling)(nil) - -func (t txdata) MarshalJSON() ([]byte, error) { - type txdata struct { - AccountNonce hexutil.Uint64 `json:"nonce" gencodec:"required"` - Price *hexutil.Big `json:"gasPrice" gencodec:"required"` - GasLimit hexutil.Uint64 `json:"gas" gencodec:"required"` - Recipient *common.Address `json:"to" rlp:"nil"` - Amount *hexutil.Big `json:"value" gencodec:"required"` - Payload hexutil.Bytes `json:"input" gencodec:"required"` - V *hexutil.Big `json:"v" gencodec:"required"` - R *hexutil.Big `json:"r" gencodec:"required"` - S *hexutil.Big `json:"s" gencodec:"required"` - Hash *common.Hash `json:"hash" rlp:"-"` - } - var enc txdata - enc.AccountNonce = hexutil.Uint64(t.AccountNonce) - enc.Price = (*hexutil.Big)(t.Price) - enc.GasLimit = hexutil.Uint64(t.GasLimit) - enc.Recipient = t.Recipient - enc.Amount = (*hexutil.Big)(t.Amount) - enc.Payload = t.Payload - enc.V = (*hexutil.Big)(t.V) - enc.R = (*hexutil.Big)(t.R) - enc.S = (*hexutil.Big)(t.S) - enc.Hash = t.Hash - return json.Marshal(&enc) -} - -func (t *txdata) UnmarshalJSON(input []byte) error { - type txdata struct { - AccountNonce *hexutil.Uint64 `json:"nonce" gencodec:"required"` - Price *hexutil.Big `json:"gasPrice" gencodec:"required"` - GasLimit *hexutil.Uint64 `json:"gas" gencodec:"required"` - Recipient *common.Address `json:"to" rlp:"nil"` - Amount *hexutil.Big `json:"value" gencodec:"required"` - Payload *hexutil.Bytes `json:"input" gencodec:"required"` - V *hexutil.Big `json:"v" gencodec:"required"` - R *hexutil.Big `json:"r" gencodec:"required"` - S *hexutil.Big `json:"s" gencodec:"required"` - Hash *common.Hash `json:"hash" rlp:"-"` - } - var dec txdata - if err := json.Unmarshal(input, &dec); err != nil { - return err - } - if dec.AccountNonce == nil { - return errors.New("missing required field 'nonce' for txdata") - } - t.AccountNonce = uint64(*dec.AccountNonce) - if dec.Price == nil { - return errors.New("missing required field 'gasPrice' for txdata") - } - t.Price = (*big.Int)(dec.Price) - if dec.GasLimit == nil { - return errors.New("missing required field 'gas' for txdata") - } - t.GasLimit = uint64(*dec.GasLimit) - if dec.Recipient != nil { - t.Recipient = dec.Recipient - } - if dec.Amount == nil { - return errors.New("missing required field 'value' for txdata") - } - t.Amount = (*big.Int)(dec.Amount) - if dec.Payload == nil { - return errors.New("missing required field 'input' for txdata") - } - t.Payload = *dec.Payload - if dec.V == nil { - return errors.New("missing required field 'v' for txdata") - } - t.V = (*big.Int)(dec.V) - if dec.R == nil { - return errors.New("missing required field 'r' for txdata") - } - t.R = (*big.Int)(dec.R) - if dec.S == nil { - return errors.New("missing required field 's' for txdata") - } - t.S = (*big.Int)(dec.S) - if dec.Hash != nil { - t.Hash = dec.Hash - } - return nil -} diff --git a/core/types/hashes.go b/core/types/hashes.go new file mode 100644 index 0000000000..28b31672af --- /dev/null +++ b/core/types/hashes.go @@ -0,0 +1,10 @@ +package types + +import ( + "github.com/tomochain/tomochain/crypto" +) + +var ( + // EmptyCodeHash is the known hash of the empty EVM bytecode. + EmptyCodeHash = crypto.Keccak256Hash(nil) // c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470 +) diff --git a/core/types/hashing.go b/core/types/hashing.go new file mode 100644 index 0000000000..8b9cb92b94 --- /dev/null +++ b/core/types/hashing.go @@ -0,0 +1,11 @@ +package types + +import ( + "bytes" + "sync" +) + +// encodeBufferPool holds temporary encoder buffers for DeriveSha and TX encoding. +var encodeBufferPool = sync.Pool{ + New: func() interface{} { return new(bytes.Buffer) }, +} diff --git a/core/types/transaction.go b/core/types/transaction.go index e2ae52d632..b3f75c5886 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -17,16 +17,20 @@ package types import ( + "bytes" "container/heap" "errors" "fmt" "io" "math/big" "sync/atomic" + "time" - "github.com/tomochain/tomochain/common" - "github.com/tomochain/tomochain/common/hexutil" "github.com/tomochain/tomochain/crypto" + + "github.com/tomochain/tomochain/common/math" + + "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/rlp" ) @@ -34,7 +38,11 @@ import ( var ( ErrInvalidSig = errors.New("invalid transaction v, r, s values") - errNoSigner = errors.New("missing signing methods") + ErrUnexpectedProtection = errors.New("transaction type does not supported EIP-155 protected signatures") + ErrInvalidTxType = errors.New("transaction type not valid in this context") + ErrTxTypeNotSupported = errors.New("transaction type not supported") + ErrGasFeeCapTooLow = errors.New("fee cap less than base fee") + errShortTypedTx = errors.New("typed transaction too short") skipNonceDestinationAddress = map[string]bool{ common.TomoXAddr: true, common.TradingStateAddr: true, @@ -52,14 +60,62 @@ func deriveSigner(V *big.Int) Signer { } } +// Transaction types. +const ( + LegacyTxType = 0x00 + AccessListTxType = 0x01 + DynamicFeeTxType = 0x02 + BlobTxType = 0x03 +) + +// Transaction is an Ethereum transaction. type Transaction struct { - data txdata + inner TxData // Consensus contents of a transaction + time time.Time // Time first seen locally (spam avoidance) + // caches hash atomic.Value size atomic.Value from atomic.Value } +// NewTx creates a new transaction. +func NewTx(inner TxData) *Transaction { + tx := new(Transaction) + tx.setDecoded(inner.copy(), 0) + return tx +} + +// TxData is the underlying data of a transaction. +// +// This is implemented by DynamicFeeTx, LegacyTx and AccessListTx. +type TxData interface { + txType() byte // returns the type ID + copy() TxData // creates a deep copy and initializes all fields + + chainID() *big.Int + accessList() AccessList + data() []byte + gas() uint64 + gasPrice() *big.Int + gasTipCap() *big.Int + gasFeeCap() *big.Int + value() *big.Int + nonce() uint64 + to() *common.Address + + rawSignatureValues() (v, r, s *big.Int) + setSignatureValues(chainID, v, r, s *big.Int) + + // effectiveGasPrice computes the gas price paid by the transaction, given + // the inclusion block baseFee. + // + // Unlike other TxData methods, the returned *big.Int should be an independent + // copy of the computed value, i.e. callers are allowed to mutate the result. + // Method implementations can use 'dst' to store the result. + effectiveGasPrice(dst *big.Int, baseFee *big.Int) *big.Int +} + type txdata struct { AccountNonce uint64 `json:"nonce" gencodec:"required"` Price *big.Int `json:"gasPrice" gencodec:"required"` @@ -77,58 +133,37 @@ type txdata struct { Hash *common.Hash `json:"hash" rlp:"-"` } -type txdataMarshaling struct { - AccountNonce hexutil.Uint64 - Price *hexutil.Big - GasLimit hexutil.Uint64 - Amount *hexutil.Big - Payload hexutil.Bytes - V *hexutil.Big - R *hexutil.Big - S *hexutil.Big -} - -func NewTransaction(nonce uint64, to common.Address, amount *big.Int, gasLimit uint64, gasPrice *big.Int, data []byte) *Transaction { - return newTransaction(nonce, &to, amount, gasLimit, gasPrice, data) -} - -func NewContractCreation(nonce uint64, amount *big.Int, gasLimit uint64, gasPrice *big.Int, data []byte) *Transaction { - return newTransaction(nonce, nil, amount, gasLimit, gasPrice, data) +// ChainId returns the EIP155 chain ID of the transaction. The return value will always be +// non-nil. For legacy transactions which are not replay-protected, the return value is +// zero. +func (tx *Transaction) ChainId() *big.Int { + return tx.inner.chainID() } -func newTransaction(nonce uint64, to *common.Address, amount *big.Int, gasLimit uint64, gasPrice *big.Int, data []byte) *Transaction { - if len(data) > 0 { - data = common.CopyBytes(data) - } - d := txdata{ - AccountNonce: nonce, - Recipient: to, - Payload: data, - Amount: new(big.Int), - GasLimit: gasLimit, - Price: new(big.Int), - V: new(big.Int), - R: new(big.Int), - S: new(big.Int), +func sanityCheckSignature(v *big.Int, r *big.Int, s *big.Int, maybeProtected bool) error { + if isProtectedV(v) && !maybeProtected { + return ErrUnexpectedProtection } - if amount != nil { - d.Amount.Set(amount) + + var plainV byte + if isProtectedV(v) { + chainID := deriveChainId(v).Uint64() + plainV = byte(v.Uint64() - 35 - 2*chainID) + } else if maybeProtected { + // Only EIP-155 signatures can be optionally protected. Since + // we determined this v value is not protected, it must be a + // raw 27 or 28. + plainV = byte(v.Uint64() - 27) + } else { + // If the signature is not optionally protected, we assume it + // must already be equal to the recovery id. + plainV = byte(v.Uint64()) } - if gasPrice != nil { - d.Price.Set(gasPrice) + if !crypto.ValidateSignatureValues(plainV, r, s, false) { + return ErrInvalidSig } - return &Transaction{data: d} -} - -// ChainId returns which chain id this transaction was signed for (if at all) -func (tx *Transaction) ChainId() *big.Int { - return deriveChainId(tx.data.V) -} - -// Protected returns whether the transaction is protected from replay protection. -func (tx *Transaction) Protected() bool { - return isProtectedV(tx.data.V) + return nil } func isProtectedV(V *big.Int) bool { @@ -140,70 +175,162 @@ func isProtectedV(V *big.Int) bool { return true } +// Protected says whether the transaction is replay-protected. +func (tx *Transaction) Protected() bool { + switch tx := tx.inner.(type) { + case *LegacyTx: + return tx.V != nil && isProtectedV(tx.V) + default: + return true + } +} + +// Type returns the transaction type. +func (tx *Transaction) Type() uint8 { + return tx.inner.txType() +} + // EncodeRLP implements rlp.Encoder func (tx *Transaction) EncodeRLP(w io.Writer) error { - return rlp.Encode(w, &tx.data) + if tx.Type() == LegacyTxType { + return rlp.Encode(w, tx.inner) + } + // It's an EIP-2718 typed TX envelope. + buf := encodeBufferPool.Get().(*bytes.Buffer) + defer encodeBufferPool.Put(buf) + buf.Reset() + if err := tx.encodeTyped(buf); err != nil { + return err + } + return rlp.Encode(w, buf.Bytes()) +} + +// encodeTyped writes the canonical encoding of a typed transaction to w. +func (tx *Transaction) encodeTyped(w *bytes.Buffer) error { + w.WriteByte(tx.Type()) + return rlp.Encode(w, tx.inner) +} + +// MarshalBinary returns the canonical encoding of the transaction. +// For legacy transactions, it returns the RLP encoding. For EIP-2718 typed +// transactions, it returns the type and payload. +func (tx *Transaction) MarshalBinary() ([]byte, error) { + if tx.Type() == LegacyTxType { + return rlp.EncodeToBytes(tx.inner) + } + var buf bytes.Buffer + err := tx.encodeTyped(&buf) + return buf.Bytes(), err } // DecodeRLP implements rlp.Decoder func (tx *Transaction) DecodeRLP(s *rlp.Stream) error { - _, size, _ := s.Kind() - err := s.Decode(&tx.data) - if err == nil { - tx.size.Store(common.StorageSize(rlp.ListSize(size))) + kind, size, err := s.Kind() + switch { + case err != nil: + return err + case kind == rlp.List: + // It's a legacy transaction. + var inner LegacyTx + err := s.Decode(&inner) + if err == nil { + tx.setDecoded(&inner, rlp.ListSize(size)) + } + return err + default: + // It's an EIP-2718 typed TX envelope. + var b []byte + if b, err = s.Bytes(); err != nil { + return err + } + inner, err := tx.decodeTyped(b) + if err == nil { + tx.setDecoded(inner, uint64(len(b))) + } + return err } - - return err } -// MarshalJSON encodes the web3 RPC transaction format. -func (tx *Transaction) MarshalJSON() ([]byte, error) { - hash := tx.Hash() - data := tx.data - data.Hash = &hash - return data.MarshalJSON() +// UnmarshalBinary decodes the canonical encoding of transactions. +// It supports legacy RLP transactions and EIP2718 typed transactions. +func (tx *Transaction) UnmarshalBinary(b []byte) error { + if len(b) > 0 && b[0] > 0x7f { + // It's a legacy transaction. + var data LegacyTx + err := rlp.DecodeBytes(b, &data) + if err != nil { + return err + } + tx.setDecoded(&data, uint64(len(b))) + return nil + } + // It's an EIP2718 typed transaction envelope. + inner, err := tx.decodeTyped(b) + if err != nil { + return err + } + tx.setDecoded(inner, uint64(len(b))) + return nil } -// UnmarshalJSON decodes the web3 RPC transaction format. -func (tx *Transaction) UnmarshalJSON(input []byte) error { - var dec txdata - if err := dec.UnmarshalJSON(input); err != nil { - return err +// decodeTyped decodes a typed transaction from the canonical format. +func (tx *Transaction) decodeTyped(b []byte) (TxData, error) { + if len(b) <= 1 { + return nil, errShortTypedTx } - var V byte - if isProtectedV(dec.V) { - chainID := deriveChainId(dec.V).Uint64() - V = byte(dec.V.Uint64() - 35 - 2*chainID) - } else { - V = byte(dec.V.Uint64() - 27) + switch b[0] { + case AccessListTxType: + var inner AccessListTx + err := rlp.DecodeBytes(b[1:], &inner) + return &inner, err + case DynamicFeeTxType: + var inner DynamicFeeTx + err := rlp.DecodeBytes(b[1:], &inner) + return &inner, err + default: + return nil, ErrTxTypeNotSupported } - if !crypto.ValidateSignatureValues(V, dec.R, dec.S, false) { - return ErrInvalidSig +} + +// setDecoded sets the inner transaction and size after decoding. +func (tx *Transaction) setDecoded(inner TxData, size uint64) { + tx.inner = inner + tx.time = time.Now() + if size > 0 { + tx.size.Store(size) } - *tx = Transaction{data: dec} - return nil } -func (tx *Transaction) Data() []byte { return common.CopyBytes(tx.data.Payload) } -func (tx *Transaction) Gas() uint64 { return tx.data.GasLimit } -func (tx *Transaction) GasPrice() *big.Int { return new(big.Int).Set(tx.data.Price) } -func (tx *Transaction) Value() *big.Int { return new(big.Int).Set(tx.data.Amount) } -func (tx *Transaction) Nonce() uint64 { return tx.data.AccountNonce } +func (tx *Transaction) Data() []byte { return common.CopyBytes(tx.inner.data()) } +func (tx *Transaction) Gas() uint64 { return tx.inner.gas() } +func (tx *Transaction) GasPrice() *big.Int { return new(big.Int).Set(tx.inner.gasPrice()) } +func (tx *Transaction) Value() *big.Int { return new(big.Int).Set(tx.inner.value()) } +func (tx *Transaction) Nonce() uint64 { return tx.inner.nonce() } func (tx *Transaction) CheckNonce() bool { return true } +// AccessList returns the access list of the transaction. +func (tx *Transaction) AccessList() AccessList { return tx.inner.accessList() } + +// GasTipCap returns the gasTipCap per gas of the transaction. +func (tx *Transaction) GasTipCap() *big.Int { return new(big.Int).Set(tx.inner.gasTipCap()) } + +// GasFeeCap returns the fee cap per gas of the transaction. +func (tx *Transaction) GasFeeCap() *big.Int { return new(big.Int).Set(tx.inner.gasFeeCap()) } + // To returns the recipient address of the transaction. // It returns nil if the transaction is a contract creation. func (tx *Transaction) To() *common.Address { - if tx.data.Recipient == nil { + if tx.inner.to() == nil { return nil } - to := *tx.data.Recipient + to := *tx.inner.to() return &to } func (tx *Transaction) From() *common.Address { - if tx.data.V != nil { - signer := deriveSigner(tx.data.V) + v, _, _ := tx.RawSignatureValues() + if v != nil { + signer := deriveSigner(v) if f, err := Sender(signer, tx); err != nil { return nil } else { @@ -237,67 +364,97 @@ func (tx *Transaction) Size() common.StorageSize { return size.(common.StorageSize) } c := writeCounter(0) - rlp.Encode(&c, &tx.data) + rlp.Encode(&c, &tx.inner) tx.size.Store(common.StorageSize(c)) return common.StorageSize(c) } -// AsMessage returns the transaction as a core.Message. -// -// AsMessage requires a signer to derive the sender. -// -// XXX Rename message to something less arbitrary? -func (tx *Transaction) AsMessage(s Signer, balanceFee *big.Int, number *big.Int) (Message, error) { - msg := Message{ - nonce: tx.data.AccountNonce, - gasLimit: tx.data.GasLimit, - gasPrice: new(big.Int).Set(tx.data.Price), - to: tx.data.Recipient, - amount: tx.data.Amount, - data: tx.data.Payload, - checkNonce: true, - balanceTokenFee: balanceFee, - } - var err error - msg.from, err = Sender(s, tx) - if balanceFee != nil { - if number.Cmp(common.TIPTRC21Fee) > 0 { - msg.gasPrice = common.TRC21GasPrice - } else { - msg.gasPrice = common.TRC21GasPriceBefore - } - } - return msg, err -} - // WithSignature returns a new transaction with the given signature. -// This signature needs to be formatted as described in the yellow paper (v+27). +// This signature needs to be in the [R || S || V] format where V is 0 or 1. func (tx *Transaction) WithSignature(signer Signer, sig []byte) (*Transaction, error) { r, s, v, err := signer.SignatureValues(tx, sig) if err != nil { return nil, err } - cpy := &Transaction{data: tx.data} - cpy.data.R, cpy.data.S, cpy.data.V = r, s, v - return cpy, nil + cpy := tx.inner.copy() + cpy.setSignatureValues(signer.ChainID(), v, r, s) + return &Transaction{inner: cpy, time: tx.time}, nil } // Cost returns amount + gasprice * gaslimit. func (tx *Transaction) Cost() *big.Int { - total := new(big.Int).Mul(tx.data.Price, new(big.Int).SetUint64(tx.data.GasLimit)) - total.Add(total, tx.data.Amount) + total := new(big.Int).Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas())) + total.Add(total, tx.Value()) return total } // Cost returns amount + gasprice * gaslimit. func (tx *Transaction) TRC21Cost() *big.Int { - total := new(big.Int).Mul(common.TRC21GasPrice, new(big.Int).SetUint64(tx.data.GasLimit)) - total.Add(total, tx.data.Amount) + total := new(big.Int).Mul(common.TRC21GasPrice, new(big.Int).SetUint64(tx.inner.gas())) + total.Add(total, tx.inner.value()) return total } func (tx *Transaction) RawSignatureValues() (*big.Int, *big.Int, *big.Int) { - return tx.data.V, tx.data.R, tx.data.S + return tx.inner.rawSignatureValues() +} + +// GasFeeCapCmp compares the fee cap of two transactions. +func (tx *Transaction) GasFeeCapCmp(other *Transaction) int { + return tx.inner.gasFeeCap().Cmp(other.inner.gasFeeCap()) +} + +// GasFeeCapIntCmp compares the fee cap of the transaction against the given fee cap. +func (tx *Transaction) GasFeeCapIntCmp(other *big.Int) int { + return tx.inner.gasFeeCap().Cmp(other) +} + +// GasTipCapCmp compares the gasTipCap of two transactions. +func (tx *Transaction) GasTipCapCmp(other *Transaction) int { + return tx.inner.gasTipCap().Cmp(other.inner.gasTipCap()) +} + +// GasTipCapIntCmp compares the gasTipCap of the transaction against the given gasTipCap. +func (tx *Transaction) GasTipCapIntCmp(other *big.Int) int { + return tx.inner.gasTipCap().Cmp(other) +} + +// EffectiveGasTip returns the effective miner gasTipCap for the given base fee. +// Note: if the effective gasTipCap is negative, this method returns both error +// the actual negative value, _and_ ErrGasFeeCapTooLow +func (tx *Transaction) EffectiveGasTip(baseFee *big.Int) (*big.Int, error) { + if baseFee == nil { + return tx.GasTipCap(), nil + } + var err error + gasFeeCap := tx.GasFeeCap() + if gasFeeCap.Cmp(baseFee) == -1 { + err = ErrGasFeeCapTooLow + } + return math.BigMin(tx.GasTipCap(), gasFeeCap.Sub(gasFeeCap, baseFee)), err +} + +// EffectiveGasTipValue is identical to EffectiveGasTip, but does not return an +// error in case the effective gasTipCap is negative +func (tx *Transaction) EffectiveGasTipValue(baseFee *big.Int) *big.Int { + effectiveTip, _ := tx.EffectiveGasTip(baseFee) + return effectiveTip +} + +// EffectiveGasTipCmp compares the effective gasTipCap of two transactions assuming the given base fee. +func (tx *Transaction) EffectiveGasTipCmp(other *Transaction, baseFee *big.Int) int { + if baseFee == nil { + return tx.GasTipCapCmp(other) + } + return tx.EffectiveGasTipValue(baseFee).Cmp(other.EffectiveGasTipValue(baseFee)) +} + +// EffectiveGasTipIntCmp compares the effective gasTipCap of a transaction to the given gasTipCap. +func (tx *Transaction) EffectiveGasTipIntCmp(other *big.Int, baseFee *big.Int) int { + if baseFee == nil { + return tx.GasTipCapIntCmp(other) + } + return tx.EffectiveGasTipValue(baseFee).Cmp(other) } func (tx *Transaction) IsSpecialTransaction() bool { @@ -466,11 +623,14 @@ func (tx *Transaction) IsTomoZApplyTransaction() bool { } func (tx *Transaction) String() string { - var from, to string - if tx.data.V != nil { + var ( + from, to string + v, r, s = tx.RawSignatureValues() + ) + if v != nil { // make a best guess about the signer and use that to derive // the sender. - signer := deriveSigner(tx.data.V) + signer := deriveSigner(v) if f, err := Sender(signer, tx); err != nil { // derive but don't cache from = "[invalid sender: invalid sig]" } else { @@ -480,12 +640,12 @@ func (tx *Transaction) String() string { from = "[invalid sender: nil V field]" } - if tx.data.Recipient == nil { + if tx.inner.to() == nil { to = "[contract creation]" } else { - to = fmt.Sprintf("%x", tx.data.Recipient[:]) + to = fmt.Sprintf("%x", tx.inner.to().Hex()) } - enc, _ := rlp.EncodeToBytes(&tx.data) + enc, _ := rlp.EncodeToBytes(&tx.inner) return fmt.Sprintf(` TX(%x) Contract: %v @@ -502,17 +662,17 @@ func (tx *Transaction) String() string { Hex: %x `, tx.Hash(), - tx.data.Recipient == nil, + tx.inner.to() == nil, from, to, - tx.data.AccountNonce, - tx.data.Price, - tx.data.GasLimit, - tx.data.Amount, - tx.data.Payload, - tx.data.V, - tx.data.R, - tx.data.S, + tx.inner.nonce(), + tx.inner.gasPrice(), + tx.inner.gas(), + tx.inner.value(), + tx.inner.data(), + v, + r, + s, enc, ) } @@ -550,49 +710,93 @@ func TxDifference(a, b Transactions) (keep Transactions) { return keep } +// HashDifference returns a new set which is the difference between a and b. +func HashDifference(a, b []common.Hash) []common.Hash { + keep := make([]common.Hash, 0, len(a)) + + remove := make(map[common.Hash]struct{}) + for _, hash := range b { + remove[hash] = struct{}{} + } + + for _, hash := range a { + if _, ok := remove[hash]; !ok { + keep = append(keep, hash) + } + } + + return keep +} + // TxByNonce implements the sort interface to allow sorting a list of transactions // by their nonces. This is usually only useful for sorting transactions from a // single account, otherwise a nonce comparison doesn't make much sense. type TxByNonce Transactions func (s TxByNonce) Len() int { return len(s) } -func (s TxByNonce) Less(i, j int) bool { return s[i].data.AccountNonce < s[j].data.AccountNonce } +func (s TxByNonce) Less(i, j int) bool { return s[i].Nonce() < s[j].Nonce() } func (s TxByNonce) Swap(i, j int) { s[i], s[j] = s[j], s[i] } -// TxByPrice implements both the sort and the heap interface, making it useful +// TxWithMinerFee wraps a transaction with its gas price or effective miner gasTipCap +type TxWithMinerFee struct { + tx *Transaction + minerFee *big.Int +} + +// NewTxWithMinerFee creates a wrapped transaction, calculating the effective +// miner gasTipCap if a base fee is provided. +// Returns error in case of a negative effective miner gasTipCap. +func NewTxWithMinerFee(tx *Transaction, baseFee *big.Int) (*TxWithMinerFee, error) { + minerFee, err := tx.EffectiveGasTip(baseFee) + if err != nil { + return nil, err + } + return &TxWithMinerFee{ + tx: tx, + minerFee: minerFee, + }, nil +} + +// TxByPriceAndTime implements both the sort and the heap interface, making it useful // for all at once sorting as well as individually adding and removing elements. -type TxByPrice struct { - txs Transactions +type TxByPriceAndTime struct { + txs []*TxWithMinerFee payersSwap map[common.Address]*big.Int } -func (s TxByPrice) Len() int { return len(s.txs) } -func (s TxByPrice) Less(i, j int) bool { - i_price := s.txs[i].data.Price - if s.txs[i].To() != nil { - if _, ok := s.payersSwap[*s.txs[i].To()]; ok { +func (s TxByPriceAndTime) Len() int { return len(s.txs) } +func (s TxByPriceAndTime) Less(i, j int) bool { + i_price := s.txs[i].minerFee + if s.txs[i].tx.To() != nil { + if _, ok := s.payersSwap[*s.txs[i].tx.To()]; ok { i_price = common.TRC21GasPrice } } - - j_price := s.txs[j].data.Price - if s.txs[j].To() != nil { - if _, ok := s.payersSwap[*s.txs[j].To()]; ok { + j_price := s.txs[j].minerFee + if s.txs[j].tx.To() != nil { + if _, ok := s.payersSwap[*s.txs[j].tx.To()]; ok { j_price = common.TRC21GasPrice } } - return i_price.Cmp(j_price) > 0 + // If the prices are equal, use the time the transaction was first seen for + // deterministic sorting + priceCmp := i_price.Cmp(j_price) > 0 + if priceCmp { + return priceCmp + } + return s.txs[i].tx.time.Before(s.txs[j].tx.time) } -func (s TxByPrice) Swap(i, j int) { s.txs[i], s.txs[j] = s.txs[j], s.txs[i] } +func (s TxByPriceAndTime) Swap(i, j int) { s.txs[i], s.txs[j] = s.txs[j], s.txs[i] } -func (s *TxByPrice) Push(x interface{}) { - s.txs = append(s.txs, x.(*Transaction)) +func (s *TxByPriceAndTime) Push(x interface{}) { + s.txs = append(s.txs, x.(*TxWithMinerFee)) } -func (s *TxByPrice) Pop() interface{} { +func (s *TxByPriceAndTime) Pop() interface{} { old := s.txs n := len(old) x := old[n-1] + old[n-1] = nil s.txs = old[0 : n-1] return x } @@ -601,9 +805,10 @@ func (s *TxByPrice) Pop() interface{} { // transactions in a profit-maximizing sorted order, while supporting removing // entire batches of transactions for non-executable accounts. type TransactionsByPriceAndNonce struct { - txs map[common.Address]Transactions // Per account nonce-sorted list of transactions - heads TxByPrice // Next transaction for each unique account (price heap) - signer Signer // Signer for the set of transactions + txs map[common.Address]Transactions // Per account nonce-sorted list of transactions + heads TxByPriceAndTime // Next transaction for each unique account (price heap) + signer Signer // Signer for the set of transactions + baseFee *big.Int // Current base fee } // NewTransactionsByPriceAndNonce creates a transaction set that can retrieve @@ -611,16 +816,22 @@ type TransactionsByPriceAndNonce struct { // // Note, the input map is reowned so the caller should not interact any more with // if after providing it to the constructor. - -// It also classifies special txs and normal txs -func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address]Transactions, signers map[common.Address]struct{}, payersSwap map[common.Address]*big.Int) (*TransactionsByPriceAndNonce, Transactions) { - // Initialize a price based heap with the head transactions - heads := TxByPrice{} - heads.payersSwap = payersSwap +func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address]Transactions, signers map[common.Address]struct{}, + payersSwap map[common.Address]*big.Int, baseFee *big.Int) (*TransactionsByPriceAndNonce, Transactions) { + // Initialize a price and received time based heap with the head transactions + heads := TxByPriceAndTime{ + txs: make([]*TxWithMinerFee, 0, len(txs)), + payersSwap: payersSwap, + } specialTxs := Transactions{} - for _, accTxs := range txs { - from, _ := Sender(signer, accTxs[0]) - var normalTxs Transactions + for from, accTxs := range txs { + acc, _ := Sender(signer, accTxs[0]) + wrapped, err := NewTxWithMinerFee(accTxs[0], baseFee) + // Remove transaction if sender doesn't match from, or if wrapping fails. + if acc != from || err != nil { + delete(txs, from) + continue + } lastSpecialTx := -1 if len(signers) > 0 { if _, ok := signers[from]; ok { @@ -635,23 +846,19 @@ func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address]Transa for i := 0; i <= lastSpecialTx; i++ { specialTxs = append(specialTxs, accTxs[i]) } - normalTxs = accTxs[lastSpecialTx+1:] } else { - normalTxs = accTxs - } - if len(normalTxs) > 0 { - heads.txs = append(heads.txs, normalTxs[0]) - // Ensure the sender address is from the signer - txs[from] = normalTxs[1:] + heads.Push(wrapped) + txs[from] = accTxs[1:] } } heap.Init(&heads) // Assemble and return the transaction set return &TransactionsByPriceAndNonce{ - txs: txs, - heads: heads, - signer: signer, + txs: txs, + heads: heads, + signer: signer, + baseFee: baseFee, }, specialTxs } @@ -660,18 +867,20 @@ func (t *TransactionsByPriceAndNonce) Peek() *Transaction { if len(t.heads.txs) == 0 { return nil } - return t.heads.txs[0] + return t.heads.txs[0].tx } // Shift replaces the current best head with the next one from the same account. func (t *TransactionsByPriceAndNonce) Shift() { - acc, _ := Sender(t.signer, t.heads.txs[0]) + acc, _ := Sender(t.signer, t.heads.txs[0].tx) if txs, ok := t.txs[acc]; ok && len(txs) > 0 { - t.heads.txs[0], t.txs[acc] = txs[0], txs[1:] - heap.Fix(&t.heads, 0) - } else { - heap.Pop(&t.heads) + if wrapped, err := NewTxWithMinerFee(txs[0], t.baseFee); err == nil { + t.heads.txs[0], t.txs[acc] = wrapped, txs[1:] + heap.Fix(&t.heads, 0) + return + } } + heap.Pop(&t.heads) } // Pop removes the best transaction, *not* replacing it with the next one from diff --git a/core/types/transaction_marshalling.go b/core/types/transaction_marshalling.go new file mode 100644 index 0000000000..6603d97af7 --- /dev/null +++ b/core/types/transaction_marshalling.go @@ -0,0 +1,277 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package types + +import ( + "encoding/json" + "errors" + "math/big" + + "github.com/tomochain/tomochain/common" + "github.com/tomochain/tomochain/common/hexutil" +) + +// txJSON is the JSON representation of transactions. +type txJSON struct { + Type hexutil.Uint64 `json:"type"` + + ChainID *hexutil.Big `json:"chainId,omitempty"` + Nonce *hexutil.Uint64 `json:"nonce"` + To *common.Address `json:"to"` + Gas *hexutil.Uint64 `json:"gas"` + GasPrice *hexutil.Big `json:"gasPrice"` + MaxPriorityFeePerGas *hexutil.Big `json:"maxPriorityFeePerGas"` + MaxFeePerGas *hexutil.Big `json:"maxFeePerGas"` + MaxFeePerDataGas *hexutil.Big `json:"maxFeePerDataGas,omitempty"` + Value *hexutil.Big `json:"value"` + Input *hexutil.Bytes `json:"input"` + AccessList *AccessList `json:"accessList,omitempty"` + BlobVersionedHashes []common.Hash `json:"blobVersionedHashes,omitempty"` + V *hexutil.Big `json:"v"` + R *hexutil.Big `json:"r"` + S *hexutil.Big `json:"s"` + + // Only used for encoding: + Hash common.Hash `json:"hash"` +} + +// MarshalJSON marshals as JSON with a hash. +func (tx *Transaction) MarshalJSON() ([]byte, error) { + var enc txJSON + // These are set for all tx types. + enc.Hash = tx.Hash() + enc.Type = hexutil.Uint64(tx.Type()) + + // Other fields are set conditionally depending on tx type. + switch itx := tx.inner.(type) { + case *LegacyTx: + enc.Nonce = (*hexutil.Uint64)(&itx.Nonce) + enc.To = tx.To() + enc.Gas = (*hexutil.Uint64)(&itx.Gas) + enc.GasPrice = (*hexutil.Big)(itx.GasPrice) + enc.Value = (*hexutil.Big)(itx.Value) + enc.Input = (*hexutil.Bytes)(&itx.Data) + enc.V = (*hexutil.Big)(itx.V) + enc.R = (*hexutil.Big)(itx.R) + enc.S = (*hexutil.Big)(itx.S) + if tx.Protected() { + enc.ChainID = (*hexutil.Big)(tx.ChainId()) + } + + case *AccessListTx: + enc.ChainID = (*hexutil.Big)(itx.ChainID) + enc.Nonce = (*hexutil.Uint64)(&itx.Nonce) + enc.To = tx.To() + enc.Gas = (*hexutil.Uint64)(&itx.Gas) + enc.GasPrice = (*hexutil.Big)(itx.GasPrice) + enc.Value = (*hexutil.Big)(itx.Value) + enc.Input = (*hexutil.Bytes)(&itx.Data) + enc.AccessList = &itx.AccessList + enc.V = (*hexutil.Big)(itx.V) + enc.R = (*hexutil.Big)(itx.R) + enc.S = (*hexutil.Big)(itx.S) + + case *DynamicFeeTx: + enc.ChainID = (*hexutil.Big)(itx.ChainID) + enc.Nonce = (*hexutil.Uint64)(&itx.Nonce) + enc.To = tx.To() + enc.Gas = (*hexutil.Uint64)(&itx.Gas) + enc.MaxFeePerGas = (*hexutil.Big)(itx.GasFeeCap) + enc.MaxPriorityFeePerGas = (*hexutil.Big)(itx.GasTipCap) + enc.Value = (*hexutil.Big)(itx.Value) + enc.Input = (*hexutil.Bytes)(&itx.Data) + enc.AccessList = &itx.AccessList + enc.V = (*hexutil.Big)(itx.V) + enc.R = (*hexutil.Big)(itx.R) + enc.S = (*hexutil.Big)(itx.S) + } + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (tx *Transaction) UnmarshalJSON(input []byte) error { + var dec txJSON + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + + // Decode / verify fields according to transaction type. + var inner TxData + switch dec.Type { + case LegacyTxType: + var itx LegacyTx + inner = &itx + if dec.Nonce == nil { + return errors.New("missing required field 'nonce' in transaction") + } + itx.Nonce = uint64(*dec.Nonce) + if dec.To != nil { + itx.To = dec.To + } + if dec.Gas == nil { + return errors.New("missing required field 'gas' in transaction") + } + itx.Gas = uint64(*dec.Gas) + if dec.GasPrice == nil { + return errors.New("missing required field 'gasPrice' in transaction") + } + itx.GasPrice = (*big.Int)(dec.GasPrice) + if dec.Value == nil { + return errors.New("missing required field 'value' in transaction") + } + itx.Value = (*big.Int)(dec.Value) + if dec.Input == nil { + return errors.New("missing required field 'input' in transaction") + } + itx.Data = *dec.Input + if dec.V == nil { + return errors.New("missing required field 'v' in transaction") + } + itx.V = (*big.Int)(dec.V) + if dec.R == nil { + return errors.New("missing required field 'r' in transaction") + } + itx.R = (*big.Int)(dec.R) + if dec.S == nil { + return errors.New("missing required field 's' in transaction") + } + itx.S = (*big.Int)(dec.S) + withSignature := itx.V.Sign() != 0 || itx.R.Sign() != 0 || itx.S.Sign() != 0 + if withSignature { + if err := sanityCheckSignature(itx.V, itx.R, itx.S, true); err != nil { + return err + } + } + + case AccessListTxType: + var itx AccessListTx + inner = &itx + if dec.ChainID == nil { + return errors.New("missing required field 'chainId' in transaction") + } + itx.ChainID = (*big.Int)(dec.ChainID) + if dec.Nonce == nil { + return errors.New("missing required field 'nonce' in transaction") + } + itx.Nonce = uint64(*dec.Nonce) + if dec.To != nil { + itx.To = dec.To + } + if dec.Gas == nil { + return errors.New("missing required field 'gas' in transaction") + } + itx.Gas = uint64(*dec.Gas) + if dec.GasPrice == nil { + return errors.New("missing required field 'gasPrice' in transaction") + } + itx.GasPrice = (*big.Int)(dec.GasPrice) + if dec.Value == nil { + return errors.New("missing required field 'value' in transaction") + } + itx.Value = (*big.Int)(dec.Value) + if dec.Input == nil { + return errors.New("missing required field 'input' in transaction") + } + itx.Data = *dec.Input + if dec.V == nil { + return errors.New("missing required field 'v' in transaction") + } + if dec.AccessList != nil { + itx.AccessList = *dec.AccessList + } + itx.V = (*big.Int)(dec.V) + if dec.R == nil { + return errors.New("missing required field 'r' in transaction") + } + itx.R = (*big.Int)(dec.R) + if dec.S == nil { + return errors.New("missing required field 's' in transaction") + } + itx.S = (*big.Int)(dec.S) + withSignature := itx.V.Sign() != 0 || itx.R.Sign() != 0 || itx.S.Sign() != 0 + if withSignature { + if err := sanityCheckSignature(itx.V, itx.R, itx.S, false); err != nil { + return err + } + } + + case DynamicFeeTxType: + var itx DynamicFeeTx + inner = &itx + if dec.ChainID == nil { + return errors.New("missing required field 'chainId' in transaction") + } + itx.ChainID = (*big.Int)(dec.ChainID) + if dec.Nonce == nil { + return errors.New("missing required field 'nonce' in transaction") + } + itx.Nonce = uint64(*dec.Nonce) + if dec.To != nil { + itx.To = dec.To + } + if dec.Gas == nil { + return errors.New("missing required field 'gas' for txdata") + } + itx.Gas = uint64(*dec.Gas) + if dec.MaxPriorityFeePerGas == nil { + return errors.New("missing required field 'maxPriorityFeePerGas' for txdata") + } + itx.GasTipCap = (*big.Int)(dec.MaxPriorityFeePerGas) + if dec.MaxFeePerGas == nil { + return errors.New("missing required field 'maxFeePerGas' for txdata") + } + itx.GasFeeCap = (*big.Int)(dec.MaxFeePerGas) + if dec.Value == nil { + return errors.New("missing required field 'value' in transaction") + } + itx.Value = (*big.Int)(dec.Value) + if dec.Input == nil { + return errors.New("missing required field 'input' in transaction") + } + itx.Data = *dec.Input + if dec.V == nil { + return errors.New("missing required field 'v' in transaction") + } + if dec.AccessList != nil { + itx.AccessList = *dec.AccessList + } + itx.V = (*big.Int)(dec.V) + if dec.R == nil { + return errors.New("missing required field 'r' in transaction") + } + itx.R = (*big.Int)(dec.R) + if dec.S == nil { + return errors.New("missing required field 's' in transaction") + } + itx.S = (*big.Int)(dec.S) + withSignature := itx.V.Sign() != 0 || itx.R.Sign() != 0 || itx.S.Sign() != 0 + if withSignature { + if err := sanityCheckSignature(itx.V, itx.R, itx.S, false); err != nil { + return err + } + } + + default: + return ErrTxTypeNotSupported + } + + // Now set the inner transaction. + tx.setDecoded(inner, 0) + + // TODO: check hash here? + return nil +} diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index 5f353a5733..37da56d26d 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -96,6 +96,7 @@ type Signer interface { // SignatureValues returns the raw R, S, V values corresponding to the // given signature. SignatureValues(tx *Transaction, sig []byte) (r, s, v *big.Int, err error) + ChainID() *big.Int // Hash returns the hash to be signed. Hash(tx *Transaction) common.Hash // Equal returns true if the given signer is the same as the receiver. @@ -117,6 +118,10 @@ func NewEIP155Signer(chainId *big.Int) EIP155Signer { } } +func (s EIP155Signer) ChainID() *big.Int { + return s.chainId +} + func (s EIP155Signer) Equal(s2 Signer) bool { eip155, ok := s2.(EIP155Signer) return ok && eip155.chainId.Cmp(s.chainId) == 0 @@ -125,24 +130,28 @@ func (s EIP155Signer) Equal(s2 Signer) bool { var big8 = big.NewInt(8) func (s EIP155Signer) Sender(tx *Transaction) (common.Address, error) { + if tx.Type() != LegacyTxType { + return common.Address{}, ErrTxTypeNotSupported + } if !tx.Protected() { return HomesteadSigner{}.Sender(tx) } if tx.ChainId().Cmp(s.chainId) != 0 { - return common.Address{}, ErrInvalidChainId + return common.Address{}, fmt.Errorf("%w: have %d want %d", ErrInvalidChainId, tx.ChainId(), s.chainId) } - V := new(big.Int).Sub(tx.data.V, s.chainIdMul) + V, R, S := tx.RawSignatureValues() + V = new(big.Int).Sub(V, s.chainIdMul) V.Sub(V, big8) - return recoverPlain(s.Hash(tx), tx.data.R, tx.data.S, V, true) + return recoverPlain(s.Hash(tx), R, S, V, true) } -// WithSignature returns a new transaction with the given signature. This signature +// SignatureValues returns signature values. This signature // needs to be in the [R || S || V] format where V is 0 or 1. func (s EIP155Signer) SignatureValues(tx *Transaction, sig []byte) (R, S, V *big.Int, err error) { - R, S, V, err = HomesteadSigner{}.SignatureValues(tx, sig) - if err != nil { - return nil, nil, nil, err + if tx.Type() != LegacyTxType { + return nil, nil, nil, ErrTxTypeNotSupported } + R, S, V = decodeSignature(sig) if s.chainId.Sign() != 0 { V = big.NewInt(int64(sig[64] + 35)) V.Add(V, s.chainIdMul) @@ -154,12 +163,12 @@ func (s EIP155Signer) SignatureValues(tx *Transaction, sig []byte) (R, S, V *big // It does not uniquely identify the transaction. func (s EIP155Signer) Hash(tx *Transaction) common.Hash { return rlpHash([]interface{}{ - tx.data.AccountNonce, - tx.data.Price, - tx.data.GasLimit, - tx.data.Recipient, - tx.data.Amount, - tx.data.Payload, + tx.Nonce(), + tx.GasPrice(), + tx.Gas(), + tx.To(), + tx.Value(), + tx.Data(), s.chainId, uint(0), uint(0), }) } @@ -168,6 +177,10 @@ func (s EIP155Signer) Hash(tx *Transaction) common.Hash { // homestead rules. type HomesteadSigner struct{ FrontierSigner } +func (s HomesteadSigner) ChainID() *big.Int { + return nil +} + func (s HomesteadSigner) Equal(s2 Signer) bool { _, ok := s2.(HomesteadSigner) return ok @@ -180,11 +193,19 @@ func (hs HomesteadSigner) SignatureValues(tx *Transaction, sig []byte) (r, s, v } func (hs HomesteadSigner) Sender(tx *Transaction) (common.Address, error) { - return recoverPlain(hs.Hash(tx), tx.data.R, tx.data.S, tx.data.V, true) + if tx.Type() != LegacyTxType { + return common.Address{}, ErrTxTypeNotSupported + } + v, r, s := tx.RawSignatureValues() + return recoverPlain(hs.Hash(tx), r, s, v, true) } type FrontierSigner struct{} +func (s FrontierSigner) ChainID() *big.Int { + return nil +} + func (s FrontierSigner) Equal(s2 Signer) bool { _, ok := s2.(FrontierSigner) return ok @@ -206,17 +227,31 @@ func (fs FrontierSigner) SignatureValues(tx *Transaction, sig []byte) (r, s, v * // It does not uniquely identify the transaction. func (fs FrontierSigner) Hash(tx *Transaction) common.Hash { return rlpHash([]interface{}{ - tx.data.AccountNonce, - tx.data.Price, - tx.data.GasLimit, - tx.data.Recipient, - tx.data.Amount, - tx.data.Payload, + tx.Nonce(), + tx.GasPrice(), + tx.Gas(), + tx.To(), + tx.Value(), + tx.Data(), }) } func (fs FrontierSigner) Sender(tx *Transaction) (common.Address, error) { - return recoverPlain(fs.Hash(tx), tx.data.R, tx.data.S, tx.data.V, false) + if tx.Type() != LegacyTxType { + return common.Address{}, ErrTxTypeNotSupported + } + v, r, s := tx.RawSignatureValues() + return recoverPlain(fs.Hash(tx), r, s, v, false) +} + +func decodeSignature(sig []byte) (r, s, v *big.Int) { + if len(sig) != crypto.SignatureLength { + panic(fmt.Sprintf("wrong size for signature: got %d, want %d", len(sig), crypto.SignatureLength)) + } + r = new(big.Int).SetBytes(sig[:32]) + s = new(big.Int).SetBytes(sig[32:64]) + v = new(big.Int).SetBytes([]byte{sig[64] + 27}) + return r, s, v } func recoverPlain(sighash common.Hash, R, S, Vb *big.Int, homestead bool) (common.Address, error) { diff --git a/core/types/tx_access_list.go b/core/types/tx_access_list.go new file mode 100644 index 0000000000..5a4a6b2417 --- /dev/null +++ b/core/types/tx_access_list.go @@ -0,0 +1,122 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package types + +import ( + "math/big" + + "github.com/tomochain/tomochain/common" +) + +//go:generate go run github.com/fjl/gencodec -type AccessTuple -out gen_access_tuple.go + +// AccessList is an EIP-2930 access list. +type AccessList []AccessTuple + +// AccessTuple is the element type of an access list. +type AccessTuple struct { + Address common.Address `json:"address" gencodec:"required"` + StorageKeys []common.Hash `json:"storageKeys" gencodec:"required"` +} + +// StorageKeys returns the total number of storage keys in the access list. +func (al AccessList) StorageKeys() int { + sum := 0 + for _, tuple := range al { + sum += len(tuple.StorageKeys) + } + return sum +} + +// AccessListTx is the data of EIP-2930 access list transactions. +type AccessListTx struct { + ChainID *big.Int // destination chain ID + Nonce uint64 // nonce of sender account + GasPrice *big.Int // wei per gas + Gas uint64 // gas limit + To *common.Address `rlp:"nil"` // nil means contract creation + Value *big.Int // wei amount + Data []byte // contract invocation input data + AccessList AccessList // EIP-2930 access list + V, R, S *big.Int // signature values +} + +// copy creates a deep copy of the transaction data and initializes all fields. +func (tx *AccessListTx) copy() TxData { + cpy := &AccessListTx{ + Nonce: tx.Nonce, + To: copyAddressPtr(tx.To), + Data: common.CopyBytes(tx.Data), + Gas: tx.Gas, + // These are copied below. + AccessList: make(AccessList, len(tx.AccessList)), + Value: new(big.Int), + ChainID: new(big.Int), + GasPrice: new(big.Int), + V: new(big.Int), + R: new(big.Int), + S: new(big.Int), + } + copy(cpy.AccessList, tx.AccessList) + if tx.Value != nil { + cpy.Value.Set(tx.Value) + } + if tx.ChainID != nil { + cpy.ChainID.Set(tx.ChainID) + } + if tx.GasPrice != nil { + cpy.GasPrice.Set(tx.GasPrice) + } + if tx.V != nil { + cpy.V.Set(tx.V) + } + if tx.R != nil { + cpy.R.Set(tx.R) + } + if tx.S != nil { + cpy.S.Set(tx.S) + } + return cpy +} + +// accessors for innerTx. +func (tx *AccessListTx) txType() byte { return AccessListTxType } +func (tx *AccessListTx) chainID() *big.Int { return tx.ChainID } +func (tx *AccessListTx) accessList() AccessList { return tx.AccessList } +func (tx *AccessListTx) data() []byte { return tx.Data } +func (tx *AccessListTx) gas() uint64 { return tx.Gas } +func (tx *AccessListTx) gasPrice() *big.Int { return tx.GasPrice } +func (tx *AccessListTx) gasTipCap() *big.Int { return tx.GasPrice } +func (tx *AccessListTx) gasFeeCap() *big.Int { return tx.GasPrice } +func (tx *AccessListTx) value() *big.Int { return tx.Value } +func (tx *AccessListTx) nonce() uint64 { return tx.Nonce } +func (tx *AccessListTx) to() *common.Address { return tx.To } +func (tx *AccessListTx) blobGas() uint64 { return 0 } +func (tx *AccessListTx) blobGasFeeCap() *big.Int { return nil } +func (tx *AccessListTx) blobHashes() []common.Hash { return nil } + +func (tx *AccessListTx) effectiveGasPrice(dst *big.Int, baseFee *big.Int) *big.Int { + return dst.Set(tx.GasPrice) +} + +func (tx *AccessListTx) rawSignatureValues() (v, r, s *big.Int) { + return tx.V, tx.R, tx.S +} + +func (tx *AccessListTx) setSignatureValues(chainID, v, r, s *big.Int) { + tx.ChainID, tx.V, tx.R, tx.S = chainID, v, r, s +} diff --git a/core/types/tx_dynamic_fee.go b/core/types/tx_dynamic_fee.go new file mode 100644 index 0000000000..4d1538d941 --- /dev/null +++ b/core/types/tx_dynamic_fee.go @@ -0,0 +1,118 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package types + +import ( + "math/big" + + "github.com/tomochain/tomochain/common" +) + +// DynamicFeeTx represents an EIP-1559 transaction. +type DynamicFeeTx struct { + ChainID *big.Int + Nonce uint64 + GasTipCap *big.Int // a.k.a. maxPriorityFeePerGas + GasFeeCap *big.Int // a.k.a. maxFeePerGas + Gas uint64 + To *common.Address `rlp:"nil"` // nil means contract creation + Value *big.Int + Data []byte + AccessList AccessList + + // Signature values + V *big.Int `json:"v" gencodec:"required"` + R *big.Int `json:"r" gencodec:"required"` + S *big.Int `json:"s" gencodec:"required"` +} + +// copy creates a deep copy of the transaction data and initializes all fields. +func (tx *DynamicFeeTx) copy() TxData { + cpy := &DynamicFeeTx{ + Nonce: tx.Nonce, + To: copyAddressPtr(tx.To), + Data: common.CopyBytes(tx.Data), + Gas: tx.Gas, + // These are copied below. + AccessList: make(AccessList, len(tx.AccessList)), + Value: new(big.Int), + ChainID: new(big.Int), + GasTipCap: new(big.Int), + GasFeeCap: new(big.Int), + V: new(big.Int), + R: new(big.Int), + S: new(big.Int), + } + copy(cpy.AccessList, tx.AccessList) + if tx.Value != nil { + cpy.Value.Set(tx.Value) + } + if tx.ChainID != nil { + cpy.ChainID.Set(tx.ChainID) + } + if tx.GasTipCap != nil { + cpy.GasTipCap.Set(tx.GasTipCap) + } + if tx.GasFeeCap != nil { + cpy.GasFeeCap.Set(tx.GasFeeCap) + } + if tx.V != nil { + cpy.V.Set(tx.V) + } + if tx.R != nil { + cpy.R.Set(tx.R) + } + if tx.S != nil { + cpy.S.Set(tx.S) + } + return cpy +} + +// accessors for innerTx. +func (tx *DynamicFeeTx) txType() byte { return DynamicFeeTxType } +func (tx *DynamicFeeTx) chainID() *big.Int { return tx.ChainID } +func (tx *DynamicFeeTx) accessList() AccessList { return tx.AccessList } +func (tx *DynamicFeeTx) data() []byte { return tx.Data } +func (tx *DynamicFeeTx) gas() uint64 { return tx.Gas } +func (tx *DynamicFeeTx) gasFeeCap() *big.Int { return tx.GasFeeCap } +func (tx *DynamicFeeTx) gasTipCap() *big.Int { return tx.GasTipCap } +func (tx *DynamicFeeTx) gasPrice() *big.Int { return tx.GasFeeCap } +func (tx *DynamicFeeTx) value() *big.Int { return tx.Value } +func (tx *DynamicFeeTx) nonce() uint64 { return tx.Nonce } +func (tx *DynamicFeeTx) to() *common.Address { return tx.To } +func (tx *DynamicFeeTx) blobGas() uint64 { return 0 } +func (tx *DynamicFeeTx) blobGasFeeCap() *big.Int { return nil } +func (tx *DynamicFeeTx) blobHashes() []common.Hash { return nil } + +func (tx *DynamicFeeTx) effectiveGasPrice(dst *big.Int, baseFee *big.Int) *big.Int { + if baseFee == nil { + return dst.Set(tx.GasFeeCap) + } + tip := dst.Sub(tx.GasFeeCap, baseFee) + if tip.Cmp(tx.GasTipCap) > 0 { + tip.Set(tx.GasTipCap) + } + return tip.Add(tip, baseFee) +} + +func (tx *DynamicFeeTx) rawSignatureValues() (v, r, s *big.Int) { + return tx.V, tx.R, tx.S +} + +func (tx *DynamicFeeTx) setSignatureValues(chainID, v, r, s *big.Int) { + tx.ChainID, tx.V, tx.R, tx.S = chainID, v, r, s +} diff --git a/core/types/tx_legacy.go b/core/types/tx_legacy.go new file mode 100644 index 0000000000..ba2b4e8dd7 --- /dev/null +++ b/core/types/tx_legacy.go @@ -0,0 +1,119 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package types + +import ( + "math/big" + + "github.com/tomochain/tomochain/common" +) + +// LegacyTx is the transaction data of the original Ethereum transactions. +type LegacyTx struct { + Nonce uint64 // nonce of sender account + GasPrice *big.Int // wei per gas + Gas uint64 // gas limit + To *common.Address `rlp:"nil"` // nil means contract creation + Value *big.Int // wei amount + Data []byte // contract invocation input data + V, R, S *big.Int // signature values +} + +// NewTransaction creates an unsigned legacy transaction. +// Deprecated: use NewTx instead. +func NewTransaction(nonce uint64, to common.Address, amount *big.Int, gasLimit uint64, gasPrice *big.Int, data []byte) *Transaction { + return NewTx(&LegacyTx{ + Nonce: nonce, + To: &to, + Value: amount, + Gas: gasLimit, + GasPrice: gasPrice, + Data: data, + }) +} + +// NewContractCreation creates an unsigned legacy transaction. +// Deprecated: use NewTx instead. +func NewContractCreation(nonce uint64, amount *big.Int, gasLimit uint64, gasPrice *big.Int, data []byte) *Transaction { + return NewTx(&LegacyTx{ + Nonce: nonce, + Value: amount, + Gas: gasLimit, + GasPrice: gasPrice, + Data: data, + }) +} + +// copy creates a deep copy of the transaction data and initializes all fields. +func (tx *LegacyTx) copy() TxData { + cpy := &LegacyTx{ + Nonce: tx.Nonce, + To: copyAddressPtr(tx.To), + Data: common.CopyBytes(tx.Data), + Gas: tx.Gas, + // These are initialized below. + Value: new(big.Int), + GasPrice: new(big.Int), + V: new(big.Int), + R: new(big.Int), + S: new(big.Int), + } + if tx.Value != nil { + cpy.Value.Set(tx.Value) + } + if tx.GasPrice != nil { + cpy.GasPrice.Set(tx.GasPrice) + } + if tx.V != nil { + cpy.V.Set(tx.V) + } + if tx.R != nil { + cpy.R.Set(tx.R) + } + if tx.S != nil { + cpy.S.Set(tx.S) + } + return cpy +} + +// accessors for innerTx. +func (tx *LegacyTx) txType() byte { return LegacyTxType } +func (tx *LegacyTx) chainID() *big.Int { return deriveChainId(tx.V) } +func (tx *LegacyTx) accessList() AccessList { return nil } +func (tx *LegacyTx) data() []byte { return tx.Data } +func (tx *LegacyTx) gas() uint64 { return tx.Gas } +func (tx *LegacyTx) gasPrice() *big.Int { return tx.GasPrice } +func (tx *LegacyTx) gasTipCap() *big.Int { return tx.GasPrice } +func (tx *LegacyTx) gasFeeCap() *big.Int { return tx.GasPrice } +func (tx *LegacyTx) value() *big.Int { return tx.Value } +func (tx *LegacyTx) nonce() uint64 { return tx.Nonce } +func (tx *LegacyTx) to() *common.Address { return tx.To } +func (tx *LegacyTx) blobGas() uint64 { return 0 } +func (tx *LegacyTx) blobGasFeeCap() *big.Int { return nil } +func (tx *LegacyTx) blobHashes() []common.Hash { return nil } + +func (tx *LegacyTx) effectiveGasPrice(dst *big.Int, baseFee *big.Int) *big.Int { + return dst.Set(tx.GasPrice) +} + +func (tx *LegacyTx) rawSignatureValues() (v, r, s *big.Int) { + return tx.V, tx.R, tx.S +} + +func (tx *LegacyTx) setSignatureValues(chainID, v, r, s *big.Int) { + tx.V, tx.R, tx.S = v, r, s +} diff --git a/crypto/crypto.go b/crypto/crypto.go index 18386f85c0..855bb4d04c 100644 --- a/crypto/crypto.go +++ b/crypto/crypto.go @@ -34,6 +34,9 @@ import ( "github.com/tomochain/tomochain/rlp" ) +// SignatureLength indicates the byte length required to carry a signature with recovery id. +const SignatureLength = 64 + 1 // 64 bytes ECDSA signature + 1 byte recovery id + var ( secp256k1_N, _ = new(big.Int).SetString("fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141", 16) secp256k1_halfN = new(big.Int).Div(secp256k1_N, big.NewInt(2)) diff --git a/ethclient/signer.go b/ethclient/signer.go index 664fdc6693..177b0ad61c 100644 --- a/ethclient/signer.go +++ b/ethclient/signer.go @@ -51,6 +51,10 @@ func (s *senderFromServer) Sender(tx *types.Transaction) (common.Address, error) return s.addr, nil } +func (s *senderFromServer) ChainID() *big.Int { + panic("can't sign with senderFromServer") +} + func (s *senderFromServer) Hash(tx *types.Transaction) common.Hash { panic("can't sign with senderFromServer") } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 33376a1071..d8710911dc 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -21,14 +21,11 @@ import ( "context" "errors" "fmt" - "github.com/tomochain/tomochain/tomoxlending/lendingstate" "math/big" "sort" "strings" "time" - "github.com/tomochain/tomochain/tomox/tradingstate" - "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/util" "github.com/tomochain/tomochain/accounts" @@ -50,6 +47,8 @@ import ( "github.com/tomochain/tomochain/params" "github.com/tomochain/tomochain/rlp" "github.com/tomochain/tomochain/rpc" + "github.com/tomochain/tomochain/tomox/tradingstate" + "github.com/tomochain/tomochain/tomoxlending/lendingstate" ) const ( @@ -355,9 +354,9 @@ func (s *PrivateAccountAPI) LockAccount(addr common.Address) bool { // signTransactions sets defaults and signs the given transaction // NOTE: the caller needs to ensure that the nonceLock is held, if applicable, // and release it after the transaction has been submitted to the tx pool -func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args SendTxArgs, passwd string) (*types.Transaction, error) { +func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args TransactionArgs, passwd string) (*types.Transaction, error) { // Look up the wallet containing the requested signer - account := accounts.Account{Address: args.From} + account := accounts.Account{Address: args.from()} wallet, err := s.am.Find(account) if err != nil { return nil, err @@ -379,12 +378,12 @@ func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args SendTxArgs // SendTransaction will create a transaction from the given arguments and // tries to sign it with the key associated with args.To. If the given passwd isn't // able to decrypt the key it fails. -func (s *PrivateAccountAPI) SendTransaction(ctx context.Context, args SendTxArgs, passwd string) (common.Hash, error) { +func (s *PrivateAccountAPI) SendTransaction(ctx context.Context, args TransactionArgs, passwd string) (common.Hash, error) { if args.Nonce == nil { // Hold the addresse's mutex around signing to prevent concurrent assignment of // the same nonce to multiple accounts. - s.nonceLock.LockAddr(args.From) - defer s.nonceLock.UnlockAddr(args.From) + s.nonceLock.LockAddr(args.from()) + defer s.nonceLock.UnlockAddr(args.from()) } signed, err := s.signTransaction(ctx, args, passwd) if err != nil { @@ -397,7 +396,7 @@ func (s *PrivateAccountAPI) SendTransaction(ctx context.Context, args SendTxArgs // tries to sign it with the key associated with args.To. If the given passwd isn't // able to decrypt the key it fails. The transaction is returned in RLP-form, not broadcast // to other nodes -func (s *PrivateAccountAPI) SignTransaction(ctx context.Context, args SendTxArgs, passwd string) (*SignTransactionResult, error) { +func (s *PrivateAccountAPI) SignTransaction(ctx context.Context, args TransactionArgs, passwd string) (*SignTransactionResult, error) { // No need to obtain the noncelock mutex, since we won't be sending this // tx into the transaction pool, but right back to the user if args.Gas == nil { @@ -424,7 +423,8 @@ func (s *PrivateAccountAPI) SignTransaction(ctx context.Context, args SendTxArgs // safely used to calculate a signature from. // // The hash is calulcated as -// keccak256("\x19Ethereum Signed Message:\n"${message length}${message}). +// +// keccak256("\x19Ethereum Signed Message:\n"${message length}${message}). // // This gives context to the signed message and prevents signing of transactions. func signHash(data []byte) []byte { @@ -488,7 +488,7 @@ func (s *PrivateAccountAPI) EcRecover(ctx context.Context, data, sig hexutil.Byt // SignAndSendTransaction was renamed to SendTransaction. This method is deprecated // and will be removed in the future. It primary goal is to give clients time to update. -func (s *PrivateAccountAPI) SignAndSendTransaction(ctx context.Context, args SendTxArgs, passwd string) (common.Hash, error) { +func (s *PrivateAccountAPI) SignAndSendTransaction(ctx context.Context, args TransactionArgs, passwd string) (common.Hash, error) { return s.SendTransaction(ctx, args, passwd) } @@ -1015,17 +1015,7 @@ func (s *PublicBlockChainAPI) getCandidatesFromSmartContract() ([]posv.Masternod return candidatesWithStakeInfo, nil } -// CallArgs represents the arguments for a call. -type CallArgs struct { - From common.Address `json:"from"` - To *common.Address `json:"to"` - Gas hexutil.Uint64 `json:"gas"` - GasPrice hexutil.Big `json:"gasPrice"` - Value hexutil.Big `json:"value"` - Data hexutil.Bytes `json:"data"` -} - -func (s *PublicBlockChainAPI) doCall(ctx context.Context, args CallArgs, blockNr rpc.BlockNumber, vmCfg vm.Config, timeout time.Duration) ([]byte, uint64, bool, error) { +func (s *PublicBlockChainAPI) doCall(ctx context.Context, args TransactionArgs, blockNr rpc.BlockNumber, vmCfg vm.Config, timeout time.Duration) ([]byte, uint64, bool, error) { defer func(start time.Time) { log.Debug("Executing EVM call finished", "runtime", time.Since(start)) }(time.Now()) statedb, header, err := s.b.StateAndHeaderByNumber(ctx, blockNr) @@ -1034,26 +1024,27 @@ func (s *PublicBlockChainAPI) doCall(ctx context.Context, args CallArgs, blockNr } // Set sender address or use a default if none specified addr := args.From - if addr == (common.Address{}) { + if *addr == (common.Address{}) { if wallets := s.b.AccountManager().Wallets(); len(wallets) > 0 { if accounts := wallets[0].Accounts(); len(accounts) > 0 { - addr = accounts[0].Address + addr = &accounts[0].Address } } } - // Set default gas & gas price if none were set - gas, gasPrice := uint64(args.Gas), args.GasPrice.ToInt() - if gas == 0 { - gas = math.MaxUint64 / 2 - } + // Set default gas price if none were set + gasPrice := args.GasPrice.ToInt() if gasPrice.Sign() == 0 { gasPrice = new(big.Int).SetUint64(defaultGasPrice) } - balanceTokenFee := big.NewInt(0).SetUint64(gas) + balanceTokenFee, _ := big.NewInt(0).SetString(args.Gas.String(), 10) balanceTokenFee = balanceTokenFee.Mul(balanceTokenFee, gasPrice) // Create new call message - msg := types.NewMessage(addr, args.To, 0, args.Value.ToInt(), gas, gasPrice, args.Data, false, balanceTokenFee) + // Get a new instance of the EVM. + msg, err := args.ToMessage(header.BaseFee, balanceTokenFee) + if err != nil { + return nil, 0, false, err + } // Setup context so it may be cancelled the call has completed // or, in case of unmetered gas, setup a context with a timeout. var cancel context.CancelFunc @@ -1103,22 +1094,23 @@ func (s *PublicBlockChainAPI) doCall(ctx context.Context, args CallArgs, blockNr // Call executes the given transaction on the state for the given block number. // It doesn't make and changes in the state/blockchain and is useful to execute and retrieve values. -func (s *PublicBlockChainAPI) Call(ctx context.Context, args CallArgs, blockNr rpc.BlockNumber) (hexutil.Bytes, error) { +func (s *PublicBlockChainAPI) Call(ctx context.Context, args TransactionArgs, blockNr rpc.BlockNumber) (hexutil.Bytes, error) { result, _, _, err := s.doCall(ctx, args, blockNr, vm.Config{}, 5*time.Second) return (hexutil.Bytes)(result), err } // EstimateGas returns an estimate of the amount of gas needed to execute the // given transaction against the current pending block. -func (s *PublicBlockChainAPI) EstimateGas(ctx context.Context, args CallArgs) (hexutil.Uint64, error) { +func (s *PublicBlockChainAPI) EstimateGas(ctx context.Context, args TransactionArgs) (hexutil.Uint64, error) { // Binary search the gas requirement, as it may be higher than the amount used var ( lo uint64 = params.TxGas - 1 hi uint64 cap uint64 ) - if uint64(args.Gas) >= params.TxGas { - hi = uint64(args.Gas) + // Determine the highest gas limit can be used during the estimation. + if args.Gas != nil && uint64(*args.Gas) >= params.TxGas { + hi = uint64(*args.Gas) } else { // Retrieve the current pending block to act as the gas ceiling block, err := s.b.BlockByNumber(ctx, rpc.LatestBlockNumber) @@ -1127,11 +1119,50 @@ func (s *PublicBlockChainAPI) EstimateGas(ctx context.Context, args CallArgs) (h } hi = block.GasLimit() } + + // Normalize the max fee per gas the call is willing to spend. + var feeCap *big.Int + if args.GasPrice != nil && (args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil) { + return 0, errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") + } else if args.GasPrice != nil { + feeCap = args.GasPrice.ToInt() + } else if args.MaxFeePerGas != nil { + feeCap = args.MaxFeePerGas.ToInt() + } else { + feeCap = common.Big0 + } + // Recap the highest gas limit with account's available balance. + if feeCap.BitLen() != 0 { + state, _, err := s.b.StateAndHeaderByNumber(ctx, rpc.LatestBlockNumber) + if err != nil { + return 0, err + } + balance := state.GetBalance(*args.From) // from can't be nil + available := new(big.Int).Set(balance) + if args.Value != nil { + if args.Value.ToInt().Cmp(available) >= 0 { + return 0, core.ErrInsufficientFundsForTransfer + } + available.Sub(available, args.Value.ToInt()) + } + allowance := new(big.Int).Div(available, feeCap) + + // If the allowance is larger than maximum uint64, skip checking + if allowance.IsUint64() && hi > allowance.Uint64() { + transfer := args.Value + if transfer == nil { + transfer = new(hexutil.Big) + } + log.Warn("Gas estimation capped by limited funds", "original", hi, "balance", balance, + "sent", transfer.ToInt(), "maxFeePerGas", feeCap, "fundable", allowance) + hi = allowance.Uint64() + } + } cap = hi // Create a helper to check if a gas allowance results in an executable transaction executable := func(gas uint64) bool { - args.Gas = hexutil.Uint64(gas) + args.Gas = (*hexutil.Uint64)(&gas) _, _, failed, err := s.doCall(ctx, args, rpc.LatestBlockNumber, vm.Config{}, 0) if err != nil || failed { @@ -1305,8 +1336,8 @@ func (s *PublicBlockChainAPI) findNearestSignedBlock(ctx context.Context, b *typ } /* - findFinalityOfBlock return finality of a block - Use blocksHashCache for to keep track - refer core/blockchain.go for more detail +findFinalityOfBlock return finality of a block +Use blocksHashCache for to keep track - refer core/blockchain.go for more detail */ func (s *PublicBlockChainAPI) findFinalityOfBlock(ctx context.Context, b *types.Block, masternodes []common.Address) (uint, error) { engine, _ := s.b.GetEngine().(*posv.Posv) @@ -1371,7 +1402,7 @@ func (s *PublicBlockChainAPI) findFinalityOfBlock(ctx context.Context, b *types. } /* - Extract signers from block +Extract signers from block */ func (s *PublicBlockChainAPI) getSigners(ctx context.Context, block *types.Block, engine *posv.Posv) ([]common.Address, error) { var err error @@ -1688,22 +1719,48 @@ func (s *PublicTransactionPoolAPI) sign(addr common.Address, tx *types.Transacti return wallet.SignTx(account, tx, chainID) } -// SendTxArgs represents the arguments to sumbit a new transaction into the transaction pool. -type SendTxArgs struct { - From common.Address `json:"from"` - To *common.Address `json:"to"` - Gas *hexutil.Uint64 `json:"gas"` - GasPrice *hexutil.Big `json:"gasPrice"` - Value *hexutil.Big `json:"value"` - Nonce *hexutil.Uint64 `json:"nonce"` +// TransactionArgs represents the arguments to construct a new transaction +// or a message call. +type TransactionArgs struct { + From *common.Address `json:"from"` + To *common.Address `json:"to"` + Gas *hexutil.Uint64 `json:"gas"` + GasPrice *hexutil.Big `json:"gasPrice"` + MaxFeePerGas *hexutil.Big `json:"maxFeePerGas"` + MaxPriorityFeePerGas *hexutil.Big `json:"maxPriorityFeePerGas"` + Value *hexutil.Big `json:"value"` + Nonce *hexutil.Uint64 `json:"nonce"` // We accept "data" and "input" for backwards-compatibility reasons. "input" is the // newer name and should be preferred by clients. Data *hexutil.Bytes `json:"data"` Input *hexutil.Bytes `json:"input"` + + // Introduced by AccessListTxType transaction. + AccessList *types.AccessList `json:"accessList,omitempty"` + ChainID *hexutil.Big `json:"chainId,omitempty"` +} + +// from retrieves the transaction sender address. +func (args *TransactionArgs) from() common.Address { + if args.From == nil { + return common.Address{} + } + return *args.From +} + +// data retrieves the transaction calldata. Input field is preferred. +func (args *TransactionArgs) data() []byte { + if args.Input != nil { + return *args.Input + } + if args.Data != nil { + return *args.Data + } + return nil } // setDefaults is a helper function that fills in default values for unspecified tx fields. -func (args *SendTxArgs) setDefaults(ctx context.Context, b Backend) error { +func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { if args.Gas == nil { args.Gas = new(hexutil.Uint64) *(*uint64)(args.Gas) = 90000 @@ -1719,7 +1776,7 @@ func (args *SendTxArgs) setDefaults(ctx context.Context, b Backend) error { args.Value = new(hexutil.Big) } if args.Nonce == nil { - nonce, err := b.GetPoolNonce(ctx, args.From) + nonce, err := b.GetPoolNonce(ctx, args.from()) if err != nil { return err } @@ -1740,20 +1797,147 @@ func (args *SendTxArgs) setDefaults(ctx context.Context, b Backend) error { return errors.New(`contract creation without any data provided`) } } + // If chain id is provided, ensure it matches the local chain id. Otherwise, set the local + // chain id as the default. + want := b.ChainConfig().ChainId + if args.ChainID != nil { + if have := (*big.Int)(args.ChainID); have.Cmp(want) != 0 { + return fmt.Errorf("chainId does not match node's (have=%v, want=%v)", have, want) + } + } else { + args.ChainID = (*hexutil.Big)(want) + } return nil } -func (args *SendTxArgs) toTransaction() *types.Transaction { - var input []byte - if args.Data != nil { - input = *args.Data - } else if args.Input != nil { - input = *args.Input +// ToMessage converts the transaction arguments to the Message type used by the +// core evm. This method is used in calls and traces that do not require a real +// live transaction. +func (args *TransactionArgs) ToMessage(baseFee *big.Int, balanceTokenFee *big.Int) (*core.Message, error) { + // Reject invalid combinations of pre- and post-1559 fee styles + if args.GasPrice != nil && (args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil) { + return nil, errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") } - if args.To == nil { - return types.NewContractCreation(uint64(*args.Nonce), (*big.Int)(args.Value), uint64(*args.Gas), (*big.Int)(args.GasPrice), input) + // Set sender address or use zero address if none specified. + addr := args.from() + + // Set default gas & gas price if none were set + var gas uint64 + if gas == 0 { + gas = uint64(math.MaxUint64 / 2) + } + if args.Gas != nil { + gas = uint64(*args.Gas) + } + var ( + gasPrice *big.Int + gasFeeCap *big.Int + gasTipCap *big.Int + ) + if baseFee == nil { + // If there's no basefee, then it must be a non-1559 execution + gasPrice = new(big.Int) + if args.GasPrice != nil { + gasPrice = args.GasPrice.ToInt() + } + gasFeeCap, gasTipCap = gasPrice, gasPrice + } else { + // A basefee is provided, necessitating 1559-type execution + if args.GasPrice != nil { + // User specified the legacy gas field, convert to 1559 gas typing + gasPrice = args.GasPrice.ToInt() + gasFeeCap, gasTipCap = gasPrice, gasPrice + } else { + // User specified 1559 gas fields (or none), use those + gasFeeCap = new(big.Int) + if args.MaxFeePerGas != nil { + gasFeeCap = args.MaxFeePerGas.ToInt() + } + gasTipCap = new(big.Int) + if args.MaxPriorityFeePerGas != nil { + gasTipCap = args.MaxPriorityFeePerGas.ToInt() + } + // Backfill the legacy gasPrice for EVM execution, unless we're all zeroes + gasPrice = new(big.Int) + if gasFeeCap.BitLen() > 0 || gasTipCap.BitLen() > 0 { + gasPrice = math.BigMin(new(big.Int).Add(gasTipCap, baseFee), gasFeeCap) + } + } } - return types.NewTransaction(uint64(*args.Nonce), *args.To, (*big.Int)(args.Value), uint64(*args.Gas), (*big.Int)(args.GasPrice), input) + value := new(big.Int) + if args.Value != nil { + value = args.Value.ToInt() + } + data := args.data() + var accessList types.AccessList + if args.AccessList != nil { + accessList = *args.AccessList + } + msg := &core.Message{ + From: addr, + To: args.To, + Value: value, + GasLimit: gas, + GasPrice: gasPrice, + GasFeeCap: gasFeeCap, + GasTipCap: gasTipCap, + Data: data, + AccessList: accessList, + SkipAccountChecks: true, + BalanceTokenFee: balanceTokenFee, + } + return msg, nil +} + +// toTransaction converts the arguments to a transaction. +// This assumes that setDefaults has been called. +func (args *TransactionArgs) toTransaction() *types.Transaction { + var data types.TxData + switch { + case args.MaxFeePerGas != nil: + al := types.AccessList{} + if args.AccessList != nil { + al = *args.AccessList + } + data = &types.DynamicFeeTx{ + To: args.To, + ChainID: (*big.Int)(args.ChainID), + Nonce: uint64(*args.Nonce), + Gas: uint64(*args.Gas), + GasFeeCap: (*big.Int)(args.MaxFeePerGas), + GasTipCap: (*big.Int)(args.MaxPriorityFeePerGas), + Value: (*big.Int)(args.Value), + Data: args.data(), + AccessList: al, + } + case args.AccessList != nil: + data = &types.AccessListTx{ + To: args.To, + ChainID: (*big.Int)(args.ChainID), + Nonce: uint64(*args.Nonce), + Gas: uint64(*args.Gas), + GasPrice: (*big.Int)(args.GasPrice), + Value: (*big.Int)(args.Value), + Data: args.data(), + AccessList: *args.AccessList, + } + default: + data = &types.LegacyTx{ + To: args.To, + Nonce: uint64(*args.Nonce), + Gas: uint64(*args.Gas), + GasPrice: (*big.Int)(args.GasPrice), + Value: (*big.Int)(args.Value), + Data: args.data(), + } + } + return types.NewTx(data) +} + +// ToTransaction converts the arguments to a transaction. +// This assumes that setDefaults has been called. +func (args *TransactionArgs) ToTransaction() *types.Transaction { + return args.toTransaction() } // submitTransaction is a helper function that submits tx to txPool and logs a message. @@ -1798,10 +1982,10 @@ func submitLendingTransaction(ctx context.Context, b Backend, tx *types.LendingT // SendTransaction creates a transaction for the given argument, sign it and submit it to the // transaction pool. -func (s *PublicTransactionPoolAPI) SendTransaction(ctx context.Context, args SendTxArgs) (common.Hash, error) { +func (s *PublicTransactionPoolAPI) SendTransaction(ctx context.Context, args TransactionArgs) (common.Hash, error) { // Look up the wallet containing the requested signer - account := accounts.Account{Address: args.From} + account := accounts.Account{Address: args.from()} wallet, err := s.b.AccountManager().Find(account) if err != nil { @@ -1811,8 +1995,8 @@ func (s *PublicTransactionPoolAPI) SendTransaction(ctx context.Context, args Sen if args.Nonce == nil { // Hold the addresse's mutex around signing to prevent concurrent assignment of // the same nonce to multiple accounts. - s.nonceLock.LockAddr(args.From) - defer s.nonceLock.UnlockAddr(args.From) + s.nonceLock.LockAddr(args.from()) + defer s.nonceLock.UnlockAddr(args.from()) } // Set some sanity defaults and terminate on failure @@ -2716,7 +2900,7 @@ type SignTransactionResult struct { // SignTransaction will sign the given transaction with the from account. // The node needs to have the private key of the account corresponding with // the given from address and it needs to be unlocked. -func (s *PublicTransactionPoolAPI) SignTransaction(ctx context.Context, args SendTxArgs) (*SignTransactionResult, error) { +func (s *PublicTransactionPoolAPI) SignTransaction(ctx context.Context, args TransactionArgs) (*SignTransactionResult, error) { if args.Gas == nil { return nil, fmt.Errorf("gas not specified") } @@ -2729,7 +2913,7 @@ func (s *PublicTransactionPoolAPI) SignTransaction(ctx context.Context, args Sen if err := args.setDefaults(ctx, s.b); err != nil { return nil, err } - tx, err := s.sign(args.From, args.toTransaction()) + tx, err := s.sign(args.from(), args.toTransaction()) if err != nil { return nil, err } @@ -2764,7 +2948,7 @@ func (s *PublicTransactionPoolAPI) PendingTransactions() ([]*RPCTransaction, err // Resend accepts an existing transaction and a new gas price and limit. It will remove // the given transaction from the pool and reinsert it with the new gas price and limit. -func (s *PublicTransactionPoolAPI) Resend(ctx context.Context, sendArgs SendTxArgs, gasPrice *hexutil.Big, gasLimit *hexutil.Uint64) (common.Hash, error) { +func (s *PublicTransactionPoolAPI) Resend(ctx context.Context, sendArgs TransactionArgs, gasPrice *hexutil.Big, gasLimit *hexutil.Uint64) (common.Hash, error) { if sendArgs.Nonce == nil { return common.Hash{}, fmt.Errorf("missing transaction nonce in transaction spec") } @@ -2784,7 +2968,7 @@ func (s *PublicTransactionPoolAPI) Resend(ctx context.Context, sendArgs SendTxAr } wantSigHash := signer.Hash(matchTx) - if pFrom, err := types.Sender(signer, p); err == nil && pFrom == sendArgs.From && signer.Hash(p) == wantSigHash { + if pFrom, err := types.Sender(signer, p); err == nil && pFrom == sendArgs.from() && signer.Hash(p) == wantSigHash { // Match. Re-sign and send the transaction. if gasPrice != nil && (*big.Int)(gasPrice).Sign() != 0 { sendArgs.GasPrice = gasPrice @@ -2792,7 +2976,7 @@ func (s *PublicTransactionPoolAPI) Resend(ctx context.Context, sendArgs SendTxAr if gasLimit != nil && *gasLimit != 0 { sendArgs.Gas = gasLimit } - signedTx, err := s.sign(sendArgs.From, sendArgs.toTransaction()) + signedTx, err := s.sign(sendArgs.from(), sendArgs.toTransaction()) if err != nil { return common.Hash{}, err } @@ -2965,7 +3149,8 @@ func GetSignersFromBlocks(b Backend, blockNumber uint64, blockHash common.Hash, // GetStakerROI Estimate ROI for stakers using the last epoc reward // then multiple by epoch per year, if the address is not masternode of last epoch - return 0 // Formular: -// ROI = average_latest_epoch_reward_for_voters*number_of_epoch_per_year/latest_total_cap*100 +// +// ROI = average_latest_epoch_reward_for_voters*number_of_epoch_per_year/latest_total_cap*100 func (s *PublicBlockChainAPI) GetStakerROI() float64 { blockNumber := s.b.CurrentBlock().Number().Uint64() lastCheckpointNumber := blockNumber - (blockNumber % s.b.ChainConfig().Posv.Epoch) - s.b.ChainConfig().Posv.Epoch // calculate for 2 epochs ago @@ -2991,7 +3176,8 @@ func (s *PublicBlockChainAPI) GetStakerROI() float64 { // GetStakerROIMasternode Estimate ROI for stakers of a specific masternode using the last epoc reward // then multiple by epoch per year, if the address is not masternode of last epoch - return 0 // Formular: -// ROI = latest_epoch_reward_for_voters*number_of_epoch_per_year/latest_total_cap*100 +// +// ROI = latest_epoch_reward_for_voters*number_of_epoch_per_year/latest_total_cap*100 func (s *PublicBlockChainAPI) GetStakerROIMasternode(masternode common.Address) float64 { votersReward := s.b.GetVotersRewards(masternode) if votersReward == nil { From 2b9b5e175a762dcf8af31f489d12922fa19a6858 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Tue, 20 Jun 2023 18:09:19 +0700 Subject: [PATCH 12/82] Fix unit tests --- core/types/transaction.go | 6 ++---- eth/tracers/tracers_test.go | 7 ++++--- les/odr_test.go | 16 ++++++++++++++-- light/odr_test.go | 2 +- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/core/types/transaction.go b/core/types/transaction.go index b3f75c5886..d390e52236 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -26,11 +26,9 @@ import ( "sync/atomic" "time" - "github.com/tomochain/tomochain/crypto" - - "github.com/tomochain/tomochain/common/math" - "github.com/tomochain/tomochain/common" + "github.com/tomochain/tomochain/common/math" + "github.com/tomochain/tomochain/crypto" "github.com/tomochain/tomochain/rlp" ) diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index 38d4075175..75ff32bb23 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -20,17 +20,18 @@ import ( "crypto/ecdsa" "crypto/rand" "encoding/json" - "github.com/tomochain/tomochain/core/rawdb" "io/ioutil" "math/big" "path/filepath" "reflect" "strings" "testing" + "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/common/hexutil" "github.com/tomochain/tomochain/common/math" "github.com/tomochain/tomochain/core" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/core/vm" "github.com/tomochain/tomochain/crypto" @@ -178,7 +179,7 @@ func TestPrestateTracerCreate2(t *testing.T) { } evm := vm.NewEVM(context, statedb, nil, params.MainnetChainConfig, vm.Config{Debug: true, Tracer: tracer}) - msg, err := tx.AsMessage(signer, nil, nil) + msg, err := core.TransactionToMessage(signer, nil, nil) if err != nil { t.Fatalf("failed to prepare transaction for tracing: %v", err) } @@ -253,7 +254,7 @@ func TestCallTracer(t *testing.T) { } evm := vm.NewEVM(context, statedb, nil, test.Genesis.Config, vm.Config{Debug: true, Tracer: tracer}) - msg, err := tx.AsMessage(signer, nil, common.Big0) + msg, err := core.TransactionToMessage(signer, nil, common.Big0) if err != nil { t.Fatalf("failed to prepare transaction for tracing: %v", err) } diff --git a/les/odr_test.go b/les/odr_test.go index 3858e34028..2f3342bbaa 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -19,7 +19,6 @@ package les import ( "bytes" "context" - "github.com/tomochain/tomochain/core/rawdb" "math/big" "testing" "time" @@ -27,6 +26,7 @@ import ( "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/common/math" "github.com/tomochain/tomochain/core" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/state" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/core/vm" @@ -110,7 +110,7 @@ func odrAccounts(ctx context.Context, db ethdb.Database, config *params.ChainCon //func TestOdrContractCallLes2(t *testing.T) { testOdr(t, 2, 2, odrContractCall) } type callmsg struct { - types.Message + core.Message } func (callmsg) CheckNonce() bool { return false } @@ -133,6 +133,18 @@ func odrContractCall(ctx context.Context, db ethdb.Database, config *params.Chai if value, ok := feeCapacity[testContractAddr]; ok { balanceTokenFee = value } + //msg := &Message{ + // To: call.To, + // From: call.From, + // Value: call.Value, + // GasLimit: call.Gas, + // GasPrice: call.GasPrice, + // GasFeeCap: call.GasFeeCap, + // GasTipCap: call.GasTipCap, + // Data: call.Data, + // AccessList: call.AccessList, + // SkipAccountChecks: false, + //} msg := callmsg{types.NewMessage(from.Address(), &testContractAddr, 0, new(big.Int), 100000, new(big.Int), data, false, balanceTokenFee)} context := core.NewEVMContext(msg, header, bc, nil) diff --git a/light/odr_test.go b/light/odr_test.go index 0c5fc78573..e2684051cc 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -148,7 +148,7 @@ func odrAccounts(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc func TestOdrContractCallLes1(t *testing.T) { testChainOdr(t, 1, odrContractCall) } type callmsg struct { - types.Message + core.Message } func (callmsg) CheckNonce() bool { return false } From 47a5cee1ab22eb76f75b20c84f13da1d6653f3bf Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 21 Jun 2023 12:53:32 +0700 Subject: [PATCH 13/82] Add transaction types to receipts --- consensus/posv/posv.go | 9 +- core/types/receipt.go | 245 +++++++++++++++++++++++++++++++++-------- 2 files changed, 205 insertions(+), 49 deletions(-) diff --git a/consensus/posv/posv.go b/consensus/posv/posv.go index 0027104970..819384b2a4 100644 --- a/consensus/posv/posv.go +++ b/consensus/posv/posv.go @@ -21,9 +21,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/tomochain/tomochain/tomox/tradingstate" - "github.com/tomochain/tomochain/tomoxlending/lendingstate" - "gopkg.in/karalabe/cookiejar.v2/collections/prque" "io/ioutil" "math/big" "math/rand" @@ -34,6 +31,10 @@ import ( "sync" "time" + "github.com/tomochain/tomochain/tomox/tradingstate" + "github.com/tomochain/tomochain/tomoxlending/lendingstate" + "gopkg.in/karalabe/cookiejar.v2/collections/prque" + lru "github.com/hashicorp/golang-lru" "github.com/tomochain/tomochain/accounts" "github.com/tomochain/tomochain/common" @@ -1146,7 +1147,7 @@ func (c *Posv) CacheData(header *types.Header, txs []*types.Transaction, receipt signTxs := []*types.Transaction{} for _, tx := range txs { if tx.IsSigningTransaction() { - var b uint + var b uint64 for _, r := range receipts { if r.TxHash == tx.Hash() { if len(r.PostState) > 0 { diff --git a/core/types/receipt.go b/core/types/receipt.go index 879aaf29c9..a92ea15fad 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -18,10 +18,15 @@ package types import ( "bytes" + "errors" "fmt" "io" + "math/big" "unsafe" + "github.com/tomochain/tomochain/crypto" + "github.com/tomochain/tomochain/params" + "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/common/hexutil" "github.com/tomochain/tomochain/rlp" @@ -34,6 +39,8 @@ var ( receiptStatusSuccessfulRLP = []byte{0x01} ) +var errShortTypedReceipt = errors.New("typed receipt too short") + const ( // ReceiptStatusFailed is the status code of a transaction if execution failed. ReceiptStatusFailed = uint64(0) @@ -45,6 +52,7 @@ const ( // Receipt represents the results of a transaction. type Receipt struct { // Consensus fields + Type uint8 `json:"type,omitempty"` PostState []byte `json:"root"` Status uint64 `json:"status"` CumulativeGasUsed uint64 `json:"cumulativeGasUsed" gencodec:"required"` @@ -52,9 +60,16 @@ type Receipt struct { Logs []*Log `json:"logs" gencodec:"required"` // Implementation fields (don't reorder!) - TxHash common.Hash `json:"transactionHash" gencodec:"required"` - ContractAddress common.Address `json:"contractAddress"` - GasUsed uint64 `json:"gasUsed" gencodec:"required"` + TxHash common.Hash `json:"transactionHash" gencodec:"required"` + ContractAddress common.Address `json:"contractAddress"` + GasUsed uint64 `json:"gasUsed" gencodec:"required"` + EffectiveGasPrice *big.Int `json:"effectiveGasPrice"` // required, but tag omitted for backwards compatibility + + // Inclusion information: These fields provide information about the inclusion of the + // transaction corresponding to this receipt. + BlockHash common.Hash `json:"blockHash,omitempty"` + BlockNumber *big.Int `json:"blockNumber,omitempty"` + TransactionIndex uint `json:"transactionIndex"` } type receiptMarshaling struct { @@ -72,14 +87,11 @@ type receiptRLP struct { Logs []*Log } -type receiptStorageRLP struct { +// storedReceiptRLP is the storage encoding of a receipt. +type storedReceiptRLP struct { PostStateOrStatus []byte CumulativeGasUsed uint64 - Bloom Bloom - TxHash common.Hash - ContractAddress common.Address - Logs []*LogForStorage - GasUsed uint64 + Logs []*Log } // NewReceipt creates a barebone transaction receipt, copying the init fields. @@ -96,21 +108,100 @@ func NewReceipt(root []byte, failed bool, cumulativeGasUsed uint64) *Receipt { // EncodeRLP implements rlp.Encoder, and flattens the consensus fields of a receipt // into an RLP stream. If no post state is present, byzantium fork is assumed. func (r *Receipt) EncodeRLP(w io.Writer) error { - return rlp.Encode(w, &receiptRLP{r.statusEncoding(), r.CumulativeGasUsed, r.Bloom, r.Logs}) + data := &receiptRLP{r.statusEncoding(), r.CumulativeGasUsed, r.Bloom, r.Logs} + if r.Type == LegacyTxType { + return rlp.Encode(w, data) + } + buf := encodeBufferPool.Get().(*bytes.Buffer) + defer encodeBufferPool.Put(buf) + buf.Reset() + if err := r.encodeTyped(data, buf); err != nil { + return err + } + return rlp.Encode(w, buf.Bytes()) +} + +// encodeTyped writes the canonical encoding of a typed receipt to w. +func (r *Receipt) encodeTyped(data *receiptRLP, w *bytes.Buffer) error { + w.WriteByte(r.Type) + return rlp.Encode(w, data) +} + +// MarshalBinary returns the consensus encoding of the receipt. +func (r *Receipt) MarshalBinary() ([]byte, error) { + if r.Type == LegacyTxType { + return rlp.EncodeToBytes(r) + } + data := &receiptRLP{r.statusEncoding(), r.CumulativeGasUsed, r.Bloom, r.Logs} + var buf bytes.Buffer + err := r.encodeTyped(data, &buf) + return buf.Bytes(), err } // DecodeRLP implements rlp.Decoder, and loads the consensus fields of a receipt // from an RLP stream. func (r *Receipt) DecodeRLP(s *rlp.Stream) error { - var dec receiptRLP - if err := s.Decode(&dec); err != nil { + kind, _, err := s.Kind() + switch { + case err != nil: return err + case kind == rlp.List: + // It's a legacy receipt. + var dec receiptRLP + if err := s.Decode(&dec); err != nil { + return err + } + r.Type = LegacyTxType + return r.setFromRLP(dec) + default: + // It's an EIP-2718 typed tx receipt. + b, err := s.Bytes() + if err != nil { + return err + } + return r.decodeTyped(b) } - if err := r.setStatus(dec.PostStateOrStatus); err != nil { - return err +} + +// UnmarshalBinary decodes the consensus encoding of receipts. +// It supports legacy RLP receipts and EIP-2718 typed receipts. +func (r *Receipt) UnmarshalBinary(b []byte) error { + if len(b) > 0 && b[0] > 0x7f { + // It's a legacy receipt decode the RLP + var data receiptRLP + err := rlp.DecodeBytes(b, &data) + if err != nil { + return err + } + r.Type = LegacyTxType + return r.setFromRLP(data) + } + // It's an EIP2718 typed transaction envelope. + return r.decodeTyped(b) +} + +// decodeTyped decodes a typed receipt from the canonical format. +func (r *Receipt) decodeTyped(b []byte) error { + if len(b) <= 1 { + return errShortTypedReceipt + } + switch b[0] { + case DynamicFeeTxType, AccessListTxType, BlobTxType: + var data receiptRLP + err := rlp.DecodeBytes(b[1:], &data) + if err != nil { + return err + } + r.Type = b[0] + return r.setFromRLP(data) + default: + return ErrTxTypeNotSupported } - r.CumulativeGasUsed, r.Bloom, r.Logs = dec.CumulativeGasUsed, dec.Bloom, dec.Logs - return nil +} + +func (r *Receipt) setFromRLP(data receiptRLP) error { + r.CumulativeGasUsed, r.Bloom, r.Logs = data.CumulativeGasUsed, data.Bloom, data.Logs + return r.setStatus(data.PostStateOrStatus) } func (r *Receipt) setStatus(postStateOrStatus []byte) error { @@ -141,7 +232,6 @@ func (r *Receipt) statusEncoding() []byte { // to approximate and limit the memory consumption of various caches. func (r *Receipt) Size() common.StorageSize { size := common.StorageSize(unsafe.Sizeof(*r)) + common.StorageSize(len(r.PostState)) - size += common.StorageSize(len(r.Logs)) * common.StorageSize(unsafe.Sizeof(Log{})) for _, log := range r.Logs { size += common.StorageSize(len(log.Topics)*common.HashLength + len(log.Data)) @@ -152,9 +242,9 @@ func (r *Receipt) Size() common.StorageSize { // String implements the Stringer interface. func (r *Receipt) String() string { if len(r.PostState) == 0 { - return fmt.Sprintf("receipt{status=%d cgas=%v bloom=%x logs=%v}", r.Status, r.CumulativeGasUsed, r.Bloom, r.Logs) + return fmt.Sprintf("receipt{type=%d status=%d cgas=%v bloom=%x logs=%v}", r.Type, r.Status, r.CumulativeGasUsed, r.Bloom, r.Logs) } - return fmt.Sprintf("receipt{med=%x cgas=%v bloom=%x logs=%v}", r.PostState, r.CumulativeGasUsed, r.Bloom, r.Logs) + return fmt.Sprintf("receipt{type=%d med=%x cgas=%v bloom=%x logs=%v}", r.Type, r.PostState, r.CumulativeGasUsed, r.Bloom, r.Logs) } // ReceiptForStorage is a wrapper around a Receipt that flattens and parses the @@ -163,48 +253,113 @@ type ReceiptForStorage Receipt // EncodeRLP implements rlp.Encoder, and flattens all content fields of a receipt // into an RLP stream. -func (r *ReceiptForStorage) EncodeRLP(w io.Writer) error { - enc := &receiptStorageRLP{ - PostStateOrStatus: (*Receipt)(r).statusEncoding(), - CumulativeGasUsed: r.CumulativeGasUsed, - Bloom: r.Bloom, - TxHash: r.TxHash, - ContractAddress: r.ContractAddress, - Logs: make([]*LogForStorage, len(r.Logs)), - GasUsed: r.GasUsed, - } - for i, log := range r.Logs { - enc.Logs[i] = (*LogForStorage)(log) +func (r *ReceiptForStorage) EncodeRLP(_w io.Writer) error { + w := rlp.NewEncoderBuffer(_w) + outerList := w.List() + w.WriteBytes((*Receipt)(r).statusEncoding()) + w.WriteUint64(r.CumulativeGasUsed) + logList := w.List() + for _, log := range r.Logs { + if err := rlp.Encode(w, log); err != nil { + return err + } } - return rlp.Encode(w, enc) + w.ListEnd(logList) + w.ListEnd(outerList) + return w.Flush() } // DecodeRLP implements rlp.Decoder, and loads both consensus and implementation // fields of a receipt from an RLP stream. func (r *ReceiptForStorage) DecodeRLP(s *rlp.Stream) error { - var dec receiptStorageRLP - if err := s.Decode(&dec); err != nil { + var stored storedReceiptRLP + if err := s.Decode(&stored); err != nil { return err } - if err := (*Receipt)(r).setStatus(dec.PostStateOrStatus); err != nil { + if err := (*Receipt)(r).setStatus(stored.PostStateOrStatus); err != nil { return err } - // Assign the consensus fields - r.CumulativeGasUsed, r.Bloom = dec.CumulativeGasUsed, dec.Bloom - r.Logs = make([]*Log, len(dec.Logs)) - for i, log := range dec.Logs { - r.Logs[i] = (*Log)(log) - } - // Assign the implementation fields - r.TxHash, r.ContractAddress, r.GasUsed = dec.TxHash, dec.ContractAddress, dec.GasUsed + r.CumulativeGasUsed = stored.CumulativeGasUsed + r.Logs = stored.Logs + r.Bloom = CreateBloom(Receipts{(*Receipt)(r)}) + return nil } -// Receipts is a wrapper around a Receipt array to implement DerivableList. +// Receipts implements DerivableList for receipts. type Receipts []*Receipt // Len returns the number of receipts in this list. -func (r Receipts) Len() int { return len(r) } +func (rs Receipts) Len() int { return len(rs) } + +// EncodeIndex encodes the i'th receipt to w. +func (rs Receipts) EncodeIndex(i int, w *bytes.Buffer) { + r := rs[i] + data := &receiptRLP{r.statusEncoding(), r.CumulativeGasUsed, r.Bloom, r.Logs} + if r.Type == LegacyTxType { + rlp.Encode(w, data) + return + } + w.WriteByte(r.Type) + switch r.Type { + case AccessListTxType, DynamicFeeTxType, BlobTxType: + rlp.Encode(w, data) + default: + // For unsupported types, write nothing. Since this is for + // DeriveSha, the error will be caught matching the derived hash + // to the block. + } +} + +// DeriveFields fills the receipts with their computed fields based on consensus +// data and contextual infos like containing block and transactions. +func (rs Receipts) DeriveFields(config *params.ChainConfig, hash common.Hash, number uint64, baseFee *big.Int, txs []*Transaction) error { + signer := MakeSigner(config, new(big.Int).SetUint64(number)) + + logIndex := uint(0) + if len(txs) != len(rs) { + return errors.New("transaction and receipt count mismatch") + } + for i := 0; i < len(rs); i++ { + // The transaction type and hash can be retrieved from the transaction itself + rs[i].Type = txs[i].Type() + rs[i].TxHash = txs[i].Hash() + + rs[i].EffectiveGasPrice = txs[i].inner.effectiveGasPrice(new(big.Int), baseFee) + + // block location fields + rs[i].BlockHash = hash + rs[i].BlockNumber = new(big.Int).SetUint64(number) + rs[i].TransactionIndex = uint(i) + + // The contract address can be derived from the transaction itself + if txs[i].To() == nil { + // Deriving the signer is expensive, only do if it's actually needed + from, _ := Sender(signer, txs[i]) + rs[i].ContractAddress = crypto.CreateAddress(from, txs[i].Nonce()) + } else { + rs[i].ContractAddress = common.Address{} + } + + // The used gas can be calculated based on previous r + if i == 0 { + rs[i].GasUsed = rs[i].CumulativeGasUsed + } else { + rs[i].GasUsed = rs[i].CumulativeGasUsed - rs[i-1].CumulativeGasUsed + } + + // The derived log fields can simply be set from the block and transaction + for j := 0; j < len(rs[i].Logs); j++ { + rs[i].Logs[j].BlockNumber = number + rs[i].Logs[j].BlockHash = hash + rs[i].Logs[j].TxHash = rs[i].TxHash + rs[i].Logs[j].TxIndex = uint(i) + rs[i].Logs[j].Index = logIndex + logIndex++ + } + } + return nil +} // GetRlp returns the RLP encoding of one receipt from the list. func (r Receipts) GetRlp(i int) []byte { From 2746f63850cf409239e447ab081e3ce4475913f1 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 21 Jun 2023 12:56:18 +0700 Subject: [PATCH 14/82] Generate JSON codec of AccessList --- core/types/gen_access_tuple.go | 43 ++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 core/types/gen_access_tuple.go diff --git a/core/types/gen_access_tuple.go b/core/types/gen_access_tuple.go new file mode 100644 index 0000000000..0d8257db6f --- /dev/null +++ b/core/types/gen_access_tuple.go @@ -0,0 +1,43 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package types + +import ( + "encoding/json" + "errors" + + "github.com/tomochain/tomochain/common" +) + +// MarshalJSON marshals as JSON. +func (a AccessTuple) MarshalJSON() ([]byte, error) { + type AccessTuple struct { + Address common.Address `json:"address" gencodec:"required"` + StorageKeys []common.Hash `json:"storageKeys" gencodec:"required"` + } + var enc AccessTuple + enc.Address = a.Address + enc.StorageKeys = a.StorageKeys + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (a *AccessTuple) UnmarshalJSON(input []byte) error { + type AccessTuple struct { + Address *common.Address `json:"address" gencodec:"required"` + StorageKeys []common.Hash `json:"storageKeys" gencodec:"required"` + } + var dec AccessTuple + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.Address == nil { + return errors.New("missing required field 'address' for AccessTuple") + } + a.Address = *dec.Address + if dec.StorageKeys == nil { + return errors.New("missing required field 'storageKeys' for AccessTuple") + } + a.StorageKeys = dec.StorageKeys + return nil +} From a08a0e419d1edcff077ae9f93fc09cbf1c1a519f Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 21 Jun 2023 17:37:43 +0700 Subject: [PATCH 15/82] Regen gencodec of some structs --- core/types/block.go | 5 +++-- core/types/gen_header_json.go | 33 +++++++++++++++++++++++++++++++-- core/types/gen_header_rlp.go | 18 +++++++++++++----- core/types/gen_log_json.go | 2 ++ core/types/gen_log_rlp.go | 7 ++----- core/types/gen_receipt_json.go | 33 +++++++++++++++++++++++++++++++++ core/types/log.go | 2 +- core/types/receipt.go | 4 ++-- 8 files changed, 87 insertions(+), 17 deletions(-) diff --git a/core/types/block.go b/core/types/block.go index d7de77b338..bf91215756 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -65,7 +65,8 @@ func (n *BlockNonce) UnmarshalText(input []byte) error { return hexutil.UnmarshalFixedText("BlockNonce", input, n[:]) } -//go:generate gencodec -type Header -field-override headerMarshaling -out gen_header_json.go +//go:generate go run github.com/fjl/gencodec -type Header -field-override headerMarshaling -out gen_header_json.go +//go:generate go run ../../rlp/rlpgen -type Header -out gen_header_rlp.go // Header represents a block header in the Ethereum blockchain. type Header struct { @@ -89,7 +90,7 @@ type Header struct { Penalties []byte `json:"penalties" gencodec:"required"` // BaseFee was added by EIP-1559 and is ignored in legacy headers. - BaseFee *big.Int `json:"baseFeePerGas"` + BaseFee *big.Int `json:"baseFeePerGas" rlp:"optional"` } // field type overrides for gencodec diff --git a/core/types/gen_header_json.go b/core/types/gen_header_json.go index e03527d801..46d17dc95d 100644 --- a/core/types/gen_header_json.go +++ b/core/types/gen_header_json.go @@ -13,6 +13,7 @@ import ( var _ = (*headerMarshaling)(nil) +// MarshalJSON marshals as JSON. func (h Header) MarshalJSON() ([]byte, error) { type Header struct { ParentHash common.Hash `json:"parentHash" gencodec:"required"` @@ -22,7 +23,7 @@ func (h Header) MarshalJSON() ([]byte, error) { TxHash common.Hash `json:"transactionsRoot" gencodec:"required"` ReceiptHash common.Hash `json:"receiptsRoot" gencodec:"required"` Bloom Bloom `json:"logsBloom" gencodec:"required"` - Difficulty *hexutil.Big `json:"difficulty" gencodec:"required"` + Difficulty *hexutil.Big `json:"difficulty" gencodec:"required"` Number *hexutil.Big `json:"number" gencodec:"required"` GasLimit hexutil.Uint64 `json:"gasLimit" gencodec:"required"` GasUsed hexutil.Uint64 `json:"gasUsed" gencodec:"required"` @@ -30,6 +31,10 @@ func (h Header) MarshalJSON() ([]byte, error) { Extra hexutil.Bytes `json:"extraData" gencodec:"required"` MixDigest common.Hash `json:"mixHash" gencodec:"required"` Nonce BlockNonce `json:"nonce" gencodec:"required"` + Validators []byte `json:"validators" gencodec:"required"` + Validator []byte `json:"validator" gencodec:"required"` + Penalties []byte `json:"penalties" gencodec:"required"` + BaseFee *hexutil.Big `json:"baseFeePerGas" rlp:"optional"` Hash common.Hash `json:"hash"` } var enc Header @@ -48,10 +53,15 @@ func (h Header) MarshalJSON() ([]byte, error) { enc.Extra = h.Extra enc.MixDigest = h.MixDigest enc.Nonce = h.Nonce + enc.Validators = h.Validators + enc.Validator = h.Validator + enc.Penalties = h.Penalties + enc.BaseFee = (*hexutil.Big)(h.BaseFee) enc.Hash = h.Hash() return json.Marshal(&enc) } +// UnmarshalJSON unmarshals from JSON. func (h *Header) UnmarshalJSON(input []byte) error { type Header struct { ParentHash *common.Hash `json:"parentHash" gencodec:"required"` @@ -61,7 +71,7 @@ func (h *Header) UnmarshalJSON(input []byte) error { TxHash *common.Hash `json:"transactionsRoot" gencodec:"required"` ReceiptHash *common.Hash `json:"receiptsRoot" gencodec:"required"` Bloom *Bloom `json:"logsBloom" gencodec:"required"` - Difficulty *hexutil.Big `json:"difficulty" gencodec:"required"` + Difficulty *hexutil.Big `json:"difficulty" gencodec:"required"` Number *hexutil.Big `json:"number" gencodec:"required"` GasLimit *hexutil.Uint64 `json:"gasLimit" gencodec:"required"` GasUsed *hexutil.Uint64 `json:"gasUsed" gencodec:"required"` @@ -69,6 +79,10 @@ func (h *Header) UnmarshalJSON(input []byte) error { Extra *hexutil.Bytes `json:"extraData" gencodec:"required"` MixDigest *common.Hash `json:"mixHash" gencodec:"required"` Nonce *BlockNonce `json:"nonce" gencodec:"required"` + Validators []byte `json:"validators" gencodec:"required"` + Validator []byte `json:"validator" gencodec:"required"` + Penalties []byte `json:"penalties" gencodec:"required"` + BaseFee *hexutil.Big `json:"baseFeePerGas" rlp:"optional"` } var dec Header if err := json.Unmarshal(input, &dec); err != nil { @@ -134,5 +148,20 @@ func (h *Header) UnmarshalJSON(input []byte) error { return errors.New("missing required field 'nonce' for Header") } h.Nonce = *dec.Nonce + if dec.Validators == nil { + return errors.New("missing required field 'validators' for Header") + } + h.Validators = dec.Validators + if dec.Validator == nil { + return errors.New("missing required field 'validator' for Header") + } + h.Validator = dec.Validator + if dec.Penalties == nil { + return errors.New("missing required field 'penalties' for Header") + } + h.Penalties = dec.Penalties + if dec.BaseFee != nil { + h.BaseFee = (*big.Int)(dec.BaseFee) + } return nil } diff --git a/core/types/gen_header_rlp.go b/core/types/gen_header_rlp.go index 1422cf6b16..e52fbbb532 100644 --- a/core/types/gen_header_rlp.go +++ b/core/types/gen_header_rlp.go @@ -5,11 +5,8 @@ package types -import ( - "io" - - "github.com/tomochain/tomochain/rlp" -) +import "github.com/tomochain/tomochain/rlp" +import "io" func (obj *Header) EncodeRLP(_w io.Writer) error { w := rlp.NewEncoderBuffer(_w) @@ -53,6 +50,17 @@ func (obj *Header) EncodeRLP(_w io.Writer) error { w.WriteBytes(obj.Validators) w.WriteBytes(obj.Validator) w.WriteBytes(obj.Penalties) + _tmp1 := obj.BaseFee != nil + if _tmp1 { + if obj.BaseFee == nil { + w.Write(rlp.EmptyString) + } else { + if obj.BaseFee.Sign() == -1 { + return rlp.ErrNegativeBigInt + } + w.WriteBigInt(obj.BaseFee) + } + } w.ListEnd(_tmp0) return w.Flush() } diff --git a/core/types/gen_log_json.go b/core/types/gen_log_json.go index 759ff8814c..ae61caf6b9 100644 --- a/core/types/gen_log_json.go +++ b/core/types/gen_log_json.go @@ -12,6 +12,7 @@ import ( var _ = (*logMarshaling)(nil) +// MarshalJSON marshals as JSON. func (l Log) MarshalJSON() ([]byte, error) { type Log struct { Address common.Address `json:"address" gencodec:"required"` @@ -37,6 +38,7 @@ func (l Log) MarshalJSON() ([]byte, error) { return json.Marshal(&enc) } +// UnmarshalJSON unmarshals from JSON. func (l *Log) UnmarshalJSON(input []byte) error { type Log struct { Address *common.Address `json:"address" gencodec:"required"` diff --git a/core/types/gen_log_rlp.go b/core/types/gen_log_rlp.go index 3f2c3ddc06..9301635297 100644 --- a/core/types/gen_log_rlp.go +++ b/core/types/gen_log_rlp.go @@ -5,11 +5,8 @@ package types -import ( - "io" - - "github.com/tomochain/tomochain/rlp" -) +import "github.com/tomochain/tomochain/rlp" +import "io" func (obj *rlpLog) EncodeRLP(_w io.Writer) error { w := rlp.NewEncoderBuffer(_w) diff --git a/core/types/gen_receipt_json.go b/core/types/gen_receipt_json.go index c698b9e36d..06f61c2e20 100644 --- a/core/types/gen_receipt_json.go +++ b/core/types/gen_receipt_json.go @@ -5,6 +5,7 @@ package types import ( "encoding/json" "errors" + "math/big" "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/common/hexutil" @@ -12,8 +13,10 @@ import ( var _ = (*receiptMarshaling)(nil) +// MarshalJSON marshals as JSON. func (r Receipt) MarshalJSON() ([]byte, error) { type Receipt struct { + Type uint8 `json:"type,omitempty"` PostState hexutil.Bytes `json:"root"` Status hexutil.Uint64 `json:"status"` CumulativeGasUsed hexutil.Uint64 `json:"cumulativeGasUsed" gencodec:"required"` @@ -22,8 +25,13 @@ func (r Receipt) MarshalJSON() ([]byte, error) { TxHash common.Hash `json:"transactionHash" gencodec:"required"` ContractAddress common.Address `json:"contractAddress"` GasUsed hexutil.Uint64 `json:"gasUsed" gencodec:"required"` + EffectiveGasPrice *big.Int `json:"effectiveGasPrice"` + BlockHash common.Hash `json:"blockHash,omitempty"` + BlockNumber *big.Int `json:"blockNumber,omitempty"` + TransactionIndex uint `json:"transactionIndex"` } var enc Receipt + enc.Type = r.Type enc.PostState = r.PostState enc.Status = hexutil.Uint64(r.Status) enc.CumulativeGasUsed = hexutil.Uint64(r.CumulativeGasUsed) @@ -32,11 +40,17 @@ func (r Receipt) MarshalJSON() ([]byte, error) { enc.TxHash = r.TxHash enc.ContractAddress = r.ContractAddress enc.GasUsed = hexutil.Uint64(r.GasUsed) + enc.EffectiveGasPrice = r.EffectiveGasPrice + enc.BlockHash = r.BlockHash + enc.BlockNumber = r.BlockNumber + enc.TransactionIndex = r.TransactionIndex return json.Marshal(&enc) } +// UnmarshalJSON unmarshals from JSON. func (r *Receipt) UnmarshalJSON(input []byte) error { type Receipt struct { + Type *uint8 `json:"type,omitempty"` PostState *hexutil.Bytes `json:"root"` Status *hexutil.Uint64 `json:"status"` CumulativeGasUsed *hexutil.Uint64 `json:"cumulativeGasUsed" gencodec:"required"` @@ -45,11 +59,18 @@ func (r *Receipt) UnmarshalJSON(input []byte) error { TxHash *common.Hash `json:"transactionHash" gencodec:"required"` ContractAddress *common.Address `json:"contractAddress"` GasUsed *hexutil.Uint64 `json:"gasUsed" gencodec:"required"` + EffectiveGasPrice *big.Int `json:"effectiveGasPrice"` + BlockHash *common.Hash `json:"blockHash,omitempty"` + BlockNumber *big.Int `json:"blockNumber,omitempty"` + TransactionIndex *uint `json:"transactionIndex"` } var dec Receipt if err := json.Unmarshal(input, &dec); err != nil { return err } + if dec.Type != nil { + r.Type = *dec.Type + } if dec.PostState != nil { r.PostState = *dec.PostState } @@ -79,5 +100,17 @@ func (r *Receipt) UnmarshalJSON(input []byte) error { return errors.New("missing required field 'gasUsed' for Receipt") } r.GasUsed = uint64(*dec.GasUsed) + if dec.EffectiveGasPrice != nil { + r.EffectiveGasPrice = dec.EffectiveGasPrice + } + if dec.BlockHash != nil { + r.BlockHash = *dec.BlockHash + } + if dec.BlockNumber != nil { + r.BlockNumber = dec.BlockNumber + } + if dec.TransactionIndex != nil { + r.TransactionIndex = *dec.TransactionIndex + } return nil } diff --git a/core/types/log.go b/core/types/log.go index bee50763a8..93567b1e61 100644 --- a/core/types/log.go +++ b/core/types/log.go @@ -25,7 +25,7 @@ import ( "github.com/tomochain/tomochain/rlp" ) -//go:generate gencodec -type Log -field-override logMarshaling -out gen_log_json.go +//go:generate go run github.com/fjl/gencodec -type Log -field-override logMarshaling -out gen_log_json.go // Log represents a contract log event. These events are generated by the LOG opcode and // stored/indexed by the node. diff --git a/core/types/receipt.go b/core/types/receipt.go index a92ea15fad..d8a197089a 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -32,7 +32,7 @@ import ( "github.com/tomochain/tomochain/rlp" ) -//go:generate gencodec -type Receipt -field-override receiptMarshaling -out gen_receipt_json.go +//go:generate go run github.com/fjl/gencodec -type Receipt -field-override receiptMarshaling -out gen_receipt_json.go var ( receiptStatusFailedRLP = []byte{} @@ -74,7 +74,7 @@ type Receipt struct { type receiptMarshaling struct { PostState hexutil.Bytes - Status hexutil.Uint + Status hexutil.Uint64 CumulativeGasUsed hexutil.Uint64 GasUsed hexutil.Uint64 } From ccdd83dc6de79c9fd456901f7b0ae35f1fecd81f Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Thu, 22 Jun 2023 21:12:18 +0700 Subject: [PATCH 16/82] Implement eip2930Signer as a wrapper to handle all transaction types --- core/types/block.go | 10 ++++ core/types/transaction_signing.go | 93 +++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/core/types/block.go b/core/types/block.go index bf91215756..0415eb7e9c 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -171,6 +171,16 @@ func rlpHash(x interface{}) (h common.Hash) { return h } +// prefixedRlpHash writes the prefix into the hasher before rlp-encoding x. +// It's used for typed transactions. +func prefixedRlpHash(prefix byte, x interface{}) (h common.Hash) { + hw := sha3.NewKeccak256() + hw.Write([]byte{prefix}) + rlp.Encode(hw, x) + hw.Sum(h[:0]) + return h +} + // Body is a simple (mutable, non-safe) data container for storing and moving // a block's data contents (transactions and uncles) together. type Body struct { diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index 37da56d26d..f6fc405506 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -103,6 +103,99 @@ type Signer interface { Equal(Signer) bool } +type eip2930Signer struct{ EIP155Signer } + +// NewEIP2930Signer returns a signer that accepts EIP-2930 access list transactions, +// EIP-155 replay protected transactions, and legacy Homestead transactions. +func NewEIP2930Signer(chainId *big.Int) Signer { + return eip2930Signer{NewEIP155Signer(chainId)} +} + +func (s eip2930Signer) ChainID() *big.Int { + return s.chainId +} + +func (s eip2930Signer) Equal(s2 Signer) bool { + x, ok := s2.(eip2930Signer) + return ok && x.chainId.Cmp(s.chainId) == 0 +} + +func (s eip2930Signer) Sender(tx *Transaction) (common.Address, error) { + V, R, S := tx.RawSignatureValues() + switch tx.Type() { + case LegacyTxType: + if !tx.Protected() { + return HomesteadSigner{}.Sender(tx) + } + V = new(big.Int).Sub(V, s.chainIdMul) + V.Sub(V, big8) + case AccessListTxType: + // AL txs are defined to use 0 and 1 as their recovery + // id, add 27 to become equivalent to unprotected Homestead signatures. + V = new(big.Int).Add(V, big.NewInt(27)) + default: + return common.Address{}, ErrTxTypeNotSupported + } + if tx.ChainId().Cmp(s.chainId) != 0 { + return common.Address{}, fmt.Errorf("%w: have %d want %d", ErrInvalidChainId, tx.ChainId(), s.chainId) + } + return recoverPlain(s.Hash(tx), R, S, V, true) +} + +func (s eip2930Signer) SignatureValues(tx *Transaction, sig []byte) (R, S, V *big.Int, err error) { + switch txdata := tx.inner.(type) { + case *LegacyTx: + return s.EIP155Signer.SignatureValues(tx, sig) + case *AccessListTx: + // Check that chain ID of tx matches the signer. We also accept ID zero here, + // because it indicates that the chain ID was not specified in the tx. + if txdata.ChainID.Sign() != 0 && txdata.ChainID.Cmp(s.chainId) != 0 { + return nil, nil, nil, fmt.Errorf("%w: have %d want %d", ErrInvalidChainId, txdata.ChainID, s.chainId) + } + R, S, _ = decodeSignature(sig) + V = big.NewInt(int64(sig[64])) + default: + return nil, nil, nil, ErrTxTypeNotSupported + } + return R, S, V, nil +} + +// Hash returns the hash to be signed by the sender. +// It does not uniquely identify the transaction. +func (s eip2930Signer) Hash(tx *Transaction) common.Hash { + switch tx.Type() { + case LegacyTxType: + return rlpHash([]interface{}{ + tx.Nonce(), + tx.GasPrice(), + tx.Gas(), + tx.To(), + tx.Value(), + tx.Data(), + s.chainId, uint(0), uint(0), + }) + case AccessListTxType: + return prefixedRlpHash( + tx.Type(), + []interface{}{ + s.chainId, + tx.Nonce(), + tx.GasPrice(), + tx.Gas(), + tx.To(), + tx.Value(), + tx.Data(), + tx.AccessList(), + }) + default: + // This _should_ not happen, but in case someone sends in a bad + // json struct via RPC, it's probably more prudent to return an + // empty hash instead of killing the node with a panic + //panic("Unsupported transaction type: %d", tx.typ) + return common.Hash{} + } +} + // EIP155Transaction implements Signer using the EIP155 rules. type EIP155Signer struct { chainId, chainIdMul *big.Int From 112df1f16cdd9f513710606dd330d40b30f6fa6f Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Fri, 23 Jun 2023 10:44:13 +0700 Subject: [PATCH 17/82] Add Berlin fork block to chain config to indicate EIP-2930 signer --- core/types/transaction_signing.go | 33 ++++++++++ core/types/transaction_signing_test.go | 5 +- core/types/transaction_test.go | 88 +++++++++++++++++--------- core/types/tx_access_list.go | 25 ++++---- core/types/tx_dynamic_fee.go | 25 ++++---- core/types/tx_legacy.go | 25 ++++---- tests/block_test_util.go | 4 +- 7 files changed, 130 insertions(+), 75 deletions(-) diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index f6fc405506..94342daf73 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -52,6 +52,39 @@ func MakeSigner(config *params.ChainConfig, blockNumber *big.Int) Signer { return signer } +// LatestSigner returns the 'most permissive' Signer available for the given chain +// configuration. Specifically, this enables support of all types of transacrions +// when their respective forks are scheduled to occur at any block number (or time) +// in the chain config. +// +// Use this in transaction-handling code where the current block number is unknown. If you +// have the current block number available, use MakeSigner instead. +func LatestSigner(config *params.ChainConfig) Signer { + if config.ChainId != nil { + if config.BerlinBlock != nil { + return NewEIP2930Signer(config.ChainId) + } + if config.EIP155Block != nil { + return NewEIP155Signer(config.ChainId) + } + } + return HomesteadSigner{} +} + +// LatestSignerForChainID returns the 'most permissive' Signer available. Specifically, +// this enables support for EIP-155 replay protection and all implemented EIP-2718 +// transaction types if chainID is non-nil. +// +// Use this in transaction-handling code where the current block number and fork +// configuration are unknown. If you have a ChainConfig, use LatestSigner instead. +// If you have a ChainConfig and know the current block number, use MakeSigner instead. +func LatestSignerForChainID(chainID *big.Int) Signer { + if chainID == nil { + return HomesteadSigner{} + } + return NewEIP2930Signer(chainID) +} + // SignTx signs the transaction using the given signer and private key func SignTx(tx *Transaction, s Signer, prv *ecdsa.PrivateKey) (*Transaction, error) { h := s.Hash(tx) diff --git a/core/types/transaction_signing_test.go b/core/types/transaction_signing_test.go index e538ee3b27..535ee13990 100644 --- a/core/types/transaction_signing_test.go +++ b/core/types/transaction_signing_test.go @@ -17,6 +17,7 @@ package types import ( + "errors" "math/big" "testing" @@ -127,8 +128,8 @@ func TestChainId(t *testing.T) { } _, err = Sender(NewEIP155Signer(big.NewInt(2)), tx) - if err != ErrInvalidChainId { - t.Error("expected error:", ErrInvalidChainId) + if !errors.Is(err, ErrInvalidChainId) { + t.Error("expected error:", ErrInvalidChainId, err) } _, err = Sender(NewEIP155Signer(big.NewInt(1)), tx) diff --git a/core/types/transaction_test.go b/core/types/transaction_test.go index bc8195c986..fcbe4d60e3 100644 --- a/core/types/transaction_test.go +++ b/core/types/transaction_test.go @@ -21,6 +21,7 @@ import ( "crypto/ecdsa" "encoding/json" "math/big" + "math/rand" "testing" "github.com/tomochain/tomochain/common" @@ -123,36 +124,77 @@ func TestRecipientNormal(t *testing.T) { } } +func TestTransactionPriceNonceSortLegacy(t *testing.T) { + testTransactionPriceNonceSort(t, nil) +} + +func TestTransactionPriceNonceSort1559(t *testing.T) { + testTransactionPriceNonceSort(t, big.NewInt(0)) + testTransactionPriceNonceSort(t, big.NewInt(5)) + testTransactionPriceNonceSort(t, big.NewInt(50)) +} + // Tests that transactions can be correctly sorted according to their price in // decreasing order, but at the same time with increasing nonces when issued by // the same account. -func TestTransactionPriceNonceSort(t *testing.T) { +func testTransactionPriceNonceSort(t *testing.T, baseFee *big.Int) { // Generate a batch of accounts to start with keys := make([]*ecdsa.PrivateKey, 25) for i := 0; i < len(keys); i++ { keys[i], _ = crypto.GenerateKey() } + signer := LatestSignerForChainID(common.Big1) - signer := HomesteadSigner{} // Generate a batch of transactions with overlapping values, but shifted nonces groups := map[common.Address]Transactions{} + expectedCount := 0 for start, key := range keys { addr := crypto.PubkeyToAddress(key.PublicKey) + count := 25 for i := 0; i < 25; i++ { - tx, _ := SignTx(NewTransaction(uint64(start+i), common.Address{}, big.NewInt(100), 100, big.NewInt(int64(start+i)), nil), signer, key) + var tx *Transaction + gasFeeCap := rand.Intn(50) + if baseFee == nil { + tx = NewTx(&LegacyTx{ + Nonce: uint64(start + i), + To: &common.Address{}, + Value: big.NewInt(100), + Gas: 100, + GasPrice: big.NewInt(int64(gasFeeCap)), + Data: nil, + }) + } else { + tx = NewTx(&DynamicFeeTx{ + Nonce: uint64(start + i), + To: &common.Address{}, + Value: big.NewInt(100), + Gas: 100, + GasFeeCap: big.NewInt(int64(gasFeeCap)), + GasTipCap: big.NewInt(int64(rand.Intn(gasFeeCap + 1))), + Data: nil, + }) + if count == 25 && int64(gasFeeCap) < baseFee.Int64() { + count = i + } + } + tx, err := SignTx(tx, signer, key) + if err != nil { + t.Fatalf("failed to sign tx: %s", err) + } groups[addr] = append(groups[addr], tx) } + expectedCount += count } - // Sort the transactions and cross check the nonce ordering - txset, _ := NewTransactionsByPriceAndNonce(signer, groups, nil, map[common.Address]*big.Int{}) + // Sort the transactions and cross-check the nonce ordering + txset, _ := NewTransactionsByPriceAndNonce(signer, groups, map[common.Address]struct{}{}, map[common.Address]*big.Int{}, baseFee) txs := Transactions{} for tx := txset.Peek(); tx != nil; tx = txset.Peek() { txs = append(txs, tx) txset.Shift() } - if len(txs) != 25*25 { - t.Errorf("expected %d transactions, found %d", 25*25, len(txs)) + if len(txs) != expectedCount { + t.Errorf("expected %d transactions, found %d", expectedCount, len(txs)) } for i, txi := range txs { fromi, _ := Sender(signer, txi) @@ -160,33 +202,21 @@ func TestTransactionPriceNonceSort(t *testing.T) { // Make sure the nonce order is valid for j, txj := range txs[i+1:] { fromj, _ := Sender(signer, txj) - if fromi == fromj && txi.Nonce() > txj.Nonce() { t.Errorf("invalid nonce ordering: tx #%d (A=%x N=%v) < tx #%d (A=%x N=%v)", i, fromi[:4], txi.Nonce(), i+j, fromj[:4], txj.Nonce()) } } - // Find the previous and next nonce of this account - prev, next := i-1, i+1 - for j := i - 1; j >= 0; j-- { - if fromj, _ := Sender(signer, txs[j]); fromi == fromj { - prev = j - break - } - } - for j := i + 1; j < len(txs); j++ { - if fromj, _ := Sender(signer, txs[j]); fromi == fromj { - next = j - break - } - } - // Make sure that in between the neighbor nonces, the transaction is correctly positioned price wise - for j := prev + 1; j < next; j++ { - fromj, _ := Sender(signer, txs[j]) - if j < i && txs[j].GasPrice().Cmp(txi.GasPrice()) < 0 { - t.Errorf("invalid gasprice ordering: tx #%d (A=%x P=%v) < tx #%d (A=%x P=%v)", j, fromj[:4], txs[j].GasPrice(), i, fromi[:4], txi.GasPrice()) + // If the next tx has different from account, the price must be lower than the current one + if i+1 < len(txs) { + next := txs[i+1] + fromNext, _ := Sender(signer, next) + tip, err := txi.EffectiveGasTip(baseFee) + nextTip, nextErr := next.EffectiveGasTip(baseFee) + if err != nil || nextErr != nil { + t.Errorf("error calculating effective tip") } - if j > i && txs[j].GasPrice().Cmp(txi.GasPrice()) > 0 { - t.Errorf("invalid gasprice ordering: tx #%d (A=%x P=%v) > tx #%d (A=%x P=%v)", j, fromj[:4], txs[j].GasPrice(), i, fromi[:4], txi.GasPrice()) + if fromi != fromNext && tip.Cmp(nextTip) < 0 { + t.Errorf("invalid gasprice ordering: tx #%d (A=%x P=%v) < tx #%d (A=%x P=%v)", i, fromi[:4], txi.GasPrice(), i+1, fromNext[:4], next.GasPrice()) } } } diff --git a/core/types/tx_access_list.go b/core/types/tx_access_list.go index 5a4a6b2417..f51af1aac1 100644 --- a/core/types/tx_access_list.go +++ b/core/types/tx_access_list.go @@ -94,20 +94,17 @@ func (tx *AccessListTx) copy() TxData { } // accessors for innerTx. -func (tx *AccessListTx) txType() byte { return AccessListTxType } -func (tx *AccessListTx) chainID() *big.Int { return tx.ChainID } -func (tx *AccessListTx) accessList() AccessList { return tx.AccessList } -func (tx *AccessListTx) data() []byte { return tx.Data } -func (tx *AccessListTx) gas() uint64 { return tx.Gas } -func (tx *AccessListTx) gasPrice() *big.Int { return tx.GasPrice } -func (tx *AccessListTx) gasTipCap() *big.Int { return tx.GasPrice } -func (tx *AccessListTx) gasFeeCap() *big.Int { return tx.GasPrice } -func (tx *AccessListTx) value() *big.Int { return tx.Value } -func (tx *AccessListTx) nonce() uint64 { return tx.Nonce } -func (tx *AccessListTx) to() *common.Address { return tx.To } -func (tx *AccessListTx) blobGas() uint64 { return 0 } -func (tx *AccessListTx) blobGasFeeCap() *big.Int { return nil } -func (tx *AccessListTx) blobHashes() []common.Hash { return nil } +func (tx *AccessListTx) txType() byte { return AccessListTxType } +func (tx *AccessListTx) chainID() *big.Int { return tx.ChainID } +func (tx *AccessListTx) accessList() AccessList { return tx.AccessList } +func (tx *AccessListTx) data() []byte { return tx.Data } +func (tx *AccessListTx) gas() uint64 { return tx.Gas } +func (tx *AccessListTx) gasPrice() *big.Int { return tx.GasPrice } +func (tx *AccessListTx) gasTipCap() *big.Int { return tx.GasPrice } +func (tx *AccessListTx) gasFeeCap() *big.Int { return tx.GasPrice } +func (tx *AccessListTx) value() *big.Int { return tx.Value } +func (tx *AccessListTx) nonce() uint64 { return tx.Nonce } +func (tx *AccessListTx) to() *common.Address { return tx.To } func (tx *AccessListTx) effectiveGasPrice(dst *big.Int, baseFee *big.Int) *big.Int { return dst.Set(tx.GasPrice) diff --git a/core/types/tx_dynamic_fee.go b/core/types/tx_dynamic_fee.go index 4d1538d941..d97c4fecb2 100644 --- a/core/types/tx_dynamic_fee.go +++ b/core/types/tx_dynamic_fee.go @@ -83,20 +83,17 @@ func (tx *DynamicFeeTx) copy() TxData { } // accessors for innerTx. -func (tx *DynamicFeeTx) txType() byte { return DynamicFeeTxType } -func (tx *DynamicFeeTx) chainID() *big.Int { return tx.ChainID } -func (tx *DynamicFeeTx) accessList() AccessList { return tx.AccessList } -func (tx *DynamicFeeTx) data() []byte { return tx.Data } -func (tx *DynamicFeeTx) gas() uint64 { return tx.Gas } -func (tx *DynamicFeeTx) gasFeeCap() *big.Int { return tx.GasFeeCap } -func (tx *DynamicFeeTx) gasTipCap() *big.Int { return tx.GasTipCap } -func (tx *DynamicFeeTx) gasPrice() *big.Int { return tx.GasFeeCap } -func (tx *DynamicFeeTx) value() *big.Int { return tx.Value } -func (tx *DynamicFeeTx) nonce() uint64 { return tx.Nonce } -func (tx *DynamicFeeTx) to() *common.Address { return tx.To } -func (tx *DynamicFeeTx) blobGas() uint64 { return 0 } -func (tx *DynamicFeeTx) blobGasFeeCap() *big.Int { return nil } -func (tx *DynamicFeeTx) blobHashes() []common.Hash { return nil } +func (tx *DynamicFeeTx) txType() byte { return DynamicFeeTxType } +func (tx *DynamicFeeTx) chainID() *big.Int { return tx.ChainID } +func (tx *DynamicFeeTx) accessList() AccessList { return tx.AccessList } +func (tx *DynamicFeeTx) data() []byte { return tx.Data } +func (tx *DynamicFeeTx) gas() uint64 { return tx.Gas } +func (tx *DynamicFeeTx) gasFeeCap() *big.Int { return tx.GasFeeCap } +func (tx *DynamicFeeTx) gasTipCap() *big.Int { return tx.GasTipCap } +func (tx *DynamicFeeTx) gasPrice() *big.Int { return tx.GasFeeCap } +func (tx *DynamicFeeTx) value() *big.Int { return tx.Value } +func (tx *DynamicFeeTx) nonce() uint64 { return tx.Nonce } +func (tx *DynamicFeeTx) to() *common.Address { return tx.To } func (tx *DynamicFeeTx) effectiveGasPrice(dst *big.Int, baseFee *big.Int) *big.Int { if baseFee == nil { diff --git a/core/types/tx_legacy.go b/core/types/tx_legacy.go index ba2b4e8dd7..c53633cfd7 100644 --- a/core/types/tx_legacy.go +++ b/core/types/tx_legacy.go @@ -91,20 +91,17 @@ func (tx *LegacyTx) copy() TxData { } // accessors for innerTx. -func (tx *LegacyTx) txType() byte { return LegacyTxType } -func (tx *LegacyTx) chainID() *big.Int { return deriveChainId(tx.V) } -func (tx *LegacyTx) accessList() AccessList { return nil } -func (tx *LegacyTx) data() []byte { return tx.Data } -func (tx *LegacyTx) gas() uint64 { return tx.Gas } -func (tx *LegacyTx) gasPrice() *big.Int { return tx.GasPrice } -func (tx *LegacyTx) gasTipCap() *big.Int { return tx.GasPrice } -func (tx *LegacyTx) gasFeeCap() *big.Int { return tx.GasPrice } -func (tx *LegacyTx) value() *big.Int { return tx.Value } -func (tx *LegacyTx) nonce() uint64 { return tx.Nonce } -func (tx *LegacyTx) to() *common.Address { return tx.To } -func (tx *LegacyTx) blobGas() uint64 { return 0 } -func (tx *LegacyTx) blobGasFeeCap() *big.Int { return nil } -func (tx *LegacyTx) blobHashes() []common.Hash { return nil } +func (tx *LegacyTx) txType() byte { return LegacyTxType } +func (tx *LegacyTx) chainID() *big.Int { return deriveChainId(tx.V) } +func (tx *LegacyTx) accessList() AccessList { return nil } +func (tx *LegacyTx) data() []byte { return tx.Data } +func (tx *LegacyTx) gas() uint64 { return tx.Gas } +func (tx *LegacyTx) gasPrice() *big.Int { return tx.GasPrice } +func (tx *LegacyTx) gasTipCap() *big.Int { return tx.GasPrice } +func (tx *LegacyTx) gasFeeCap() *big.Int { return tx.GasPrice } +func (tx *LegacyTx) value() *big.Int { return tx.Value } +func (tx *LegacyTx) nonce() uint64 { return tx.Nonce } +func (tx *LegacyTx) to() *common.Address { return tx.To } func (tx *LegacyTx) effectiveGasPrice(dst *big.Int, baseFee *big.Int) *big.Int { return dst.Set(tx.GasPrice) diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 1bfd053d45..b07f23b11d 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -22,7 +22,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "github.com/tomochain/tomochain/core/rawdb" "math/big" "github.com/tomochain/tomochain/common" @@ -30,6 +29,7 @@ import ( "github.com/tomochain/tomochain/common/math" "github.com/tomochain/tomochain/consensus/ethash" "github.com/tomochain/tomochain/core" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/state" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/core/vm" @@ -61,7 +61,7 @@ type btBlock struct { UncleHeaders []*btHeader } -//go:generate gencodec -type btHeader -field-override btHeaderMarshaling -out gen_btheader.go +//go:generate go run github.com/fjl/gencodec -type btHeader -field-override btHeaderMarshaling -out gen_btheader.go type btHeader struct { Bloom types.Bloom From baf77139d00e8dfa95248d1d6c2d88751e2d9502 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Fri, 23 Jun 2023 11:15:39 +0700 Subject: [PATCH 18/82] Implement londonSigner to handle dynamicFeeTx --- core/types/transaction_signing.go | 72 +++++++++++++++++++++++++++++-- params/config.go | 9 ++-- 2 files changed, 74 insertions(+), 7 deletions(-) diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index 94342daf73..17ccdcf8ca 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -61,8 +61,8 @@ func MakeSigner(config *params.ChainConfig, blockNumber *big.Int) Signer { // have the current block number available, use MakeSigner instead. func LatestSigner(config *params.ChainConfig) Signer { if config.ChainId != nil { - if config.BerlinBlock != nil { - return NewEIP2930Signer(config.ChainId) + if config.LondonBlock != nil { + return NewLondonSigner(config.ChainId) } if config.EIP155Block != nil { return NewEIP155Signer(config.ChainId) @@ -82,7 +82,7 @@ func LatestSignerForChainID(chainID *big.Int) Signer { if chainID == nil { return HomesteadSigner{} } - return NewEIP2930Signer(chainID) + return NewLondonSigner(chainID) } // SignTx signs the transaction using the given signer and private key @@ -136,6 +136,72 @@ type Signer interface { Equal(Signer) bool } +type londonSigner struct{ eip2930Signer } + +// NewLondonSigner returns a signer that accepts +// - EIP-1559 dynamic fee transactions +// - EIP-2930 access list transactions, +// - EIP-155 replay protected transactions, and +// - legacy Homestead transactions. +func NewLondonSigner(chainId *big.Int) Signer { + return londonSigner{eip2930Signer{NewEIP155Signer(chainId)}} +} + +func (s londonSigner) Sender(tx *Transaction) (common.Address, error) { + if tx.Type() != DynamicFeeTxType { + return s.eip2930Signer.Sender(tx) + } + V, R, S := tx.RawSignatureValues() + // DynamicFee txs are defined to use 0 and 1 as their recovery + // id, add 27 to become equivalent to unprotected Homestead signatures. + V = new(big.Int).Add(V, big.NewInt(27)) + if tx.ChainId().Cmp(s.chainId) != 0 { + return common.Address{}, fmt.Errorf("%w: have %d want %d", ErrInvalidChainId, tx.ChainId(), s.chainId) + } + return recoverPlain(s.Hash(tx), R, S, V, true) +} + +func (s londonSigner) Equal(s2 Signer) bool { + x, ok := s2.(londonSigner) + return ok && x.chainId.Cmp(s.chainId) == 0 +} + +func (s londonSigner) SignatureValues(tx *Transaction, sig []byte) (R, S, V *big.Int, err error) { + txdata, ok := tx.inner.(*DynamicFeeTx) + if !ok { + return s.eip2930Signer.SignatureValues(tx, sig) + } + // Check that chain ID of tx matches the signer. We also accept ID zero here, + // because it indicates that the chain ID was not specified in the tx. + if txdata.ChainID.Sign() != 0 && txdata.ChainID.Cmp(s.chainId) != 0 { + return nil, nil, nil, fmt.Errorf("%w: have %d want %d", ErrInvalidChainId, txdata.ChainID, s.chainId) + } + R, S, _ = decodeSignature(sig) + V = big.NewInt(int64(sig[64])) + return R, S, V, nil +} + +// Hash returns the hash to be signed by the sender. +// It does not uniquely identify the transaction. +func (s londonSigner) Hash(tx *Transaction) common.Hash { + if tx.Type() != DynamicFeeTxType { + return s.eip2930Signer.Hash(tx) + } + return prefixedRlpHash( + tx.Type(), + []interface{}{ + s.chainId, + tx.Nonce(), + tx.GasTipCap(), + tx.GasFeeCap(), + tx.Gas(), + tx.To(), + tx.Value(), + tx.Data(), + tx.AccessList(), + }) +} + type eip2930Signer struct{ EIP155Signer } // NewEIP2930Signer returns a signer that accepts EIP-2930 access list transactions, diff --git a/params/config.go b/params/config.go index 056457d9e8..7649e266d8 100644 --- a/params/config.go +++ b/params/config.go @@ -102,16 +102,16 @@ var ( // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil, nil} + AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, big.NewInt(0), new(EthashConfig), nil, nil} // AllPosvProtocolChanges contains every protocol change (EIPs) introduced // and accepted by the Ethereum core developers into the Posv consensus. // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllPosvProtocolChanges = &ChainConfig{big.NewInt(89), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, &PosvConfig{Period: 0, Epoch: 30000}} - AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}, nil} - TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil, nil} + AllPosvProtocolChanges = &ChainConfig{big.NewInt(89), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, big.NewInt(0), nil, nil, &PosvConfig{Period: 0, Epoch: 30000}} + AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, big.NewInt(0), nil, &CliqueConfig{Period: 0, Epoch: 30000}, nil} + TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, big.NewInt(0), new(EthashConfig), nil, nil} TestRules = TestChainConfig.Rules(new(big.Int)) ) @@ -137,6 +137,7 @@ type ChainConfig struct { ByzantiumBlock *big.Int `json:"byzantiumBlock,omitempty"` // Byzantium switch block (nil = no fork, 0 = already on byzantium) ConstantinopleBlock *big.Int `json:"constantinopleBlock,omitempty"` // Constantinople switch block (nil = no fork, 0 = already activated) + LondonBlock *big.Int `json:"londonBlock,omitempty"` // Berlin switch block (nil = no fork, 0 = already on berlin) // Various consensus engines Ethash *EthashConfig `json:"ethash,omitempty"` From e5f339037753e3b1fbbe436b81be3f7ec21be6bd Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Fri, 23 Jun 2023 17:23:35 +0700 Subject: [PATCH 19/82] Check fork block --- params/config.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/params/config.go b/params/config.go index 7649e266d8..339a9f8aca 100644 --- a/params/config.go +++ b/params/config.go @@ -246,6 +246,11 @@ func (c *ChainConfig) IsIstanbul(num *big.Int) bool { return isForked(common.TIPTomoXCancellationFee, num) } +// IsLondon returns whether num is either equal to the London fork block or greater. +func (c *ChainConfig) IsLondon(num *big.Int) bool { + return isBlockForked(c.LondonBlock, num) +} + func (c *ChainConfig) IsTIP2019(num *big.Int) bool { return isForked(common.TIP2019Block, num) } @@ -354,6 +359,16 @@ func isForked(s, head *big.Int) bool { return s.Cmp(head) <= 0 } +// isBlockForked returns whether a fork scheduled at block s is active at the +// given head block. Whilst this method is the same as isTimestampForked, they +// are explicitly separate for clearer reading. +func isBlockForked(s, head *big.Int) bool { + if s == nil || head == nil { + return false + } + return s.Cmp(head) <= 0 +} + func configNumEqual(x, y *big.Int) bool { if x == nil { return y == nil From b78e67e016da9f1d1dcf18f4e09087942476cc79 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Fri, 23 Jun 2023 17:36:47 +0700 Subject: [PATCH 20/82] Fix comparing TxByPriceAndTime --- core/types/block_test.go | 4 ++-- core/types/transaction.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/types/block_test.go b/core/types/block_test.go index 9b78b653c7..4f33982d20 100644 --- a/core/types/block_test.go +++ b/core/types/block_test.go @@ -17,13 +17,13 @@ package types import ( + "bytes" "math/big" + "reflect" "testing" - "bytes" "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/rlp" - "reflect" ) // from bcValidBlockTest.json, "SimpleTx" diff --git a/core/types/transaction.go b/core/types/transaction.go index d390e52236..c06bf08914 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -778,9 +778,9 @@ func (s TxByPriceAndTime) Less(i, j int) bool { } // If the prices are equal, use the time the transaction was first seen for // deterministic sorting - priceCmp := i_price.Cmp(j_price) > 0 - if priceCmp { - return priceCmp + priceCmp := i_price.Cmp(j_price) + if priceCmp != 0 { + return priceCmp > 0 } return s.txs[i].tx.time.Before(s.txs[j].tx.time) } From fc3d7cfe0c656fa8a75b7515e6a1cd52ff230aa8 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Fri, 23 Jun 2023 18:06:28 +0700 Subject: [PATCH 21/82] Add block unit tests --- core/types/block.go | 18 +++++ core/types/block_test.go | 128 ++++++++++++++++++++++++++++++++++++ eth/tracers/tracers_test.go | 4 +- 3 files changed, 148 insertions(+), 2 deletions(-) diff --git a/core/types/block.go b/core/types/block.go index 0415eb7e9c..99b051dc32 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -511,3 +511,21 @@ func (self blockSorter) Swap(i, j int) { func (self blockSorter) Less(i, j int) bool { return self.by(self.blocks[i], self.blocks[j]) } func Number(b1, b2 *Block) bool { return b1.header.Number.Cmp(b2.header.Number) < 0 } + +// HeaderParentHashFromRLP returns the parentHash of an RLP-encoded +// header. If 'header' is invalid, the zero hash is returned. +func HeaderParentHashFromRLP(header []byte) common.Hash { + // parentHash is the first list element. + listContent, _, err := rlp.SplitList(header) + if err != nil { + return common.Hash{} + } + parentHash, _, err := rlp.SplitString(listContent) + if err != nil { + return common.Hash{} + } + if len(parentHash) != 32 { + return common.Hash{} + } + return common.BytesToHash(parentHash) +} diff --git a/core/types/block_test.go b/core/types/block_test.go index 4f33982d20..fddd279f28 100644 --- a/core/types/block_test.go +++ b/core/types/block_test.go @@ -23,6 +23,9 @@ import ( "testing" "github.com/tomochain/tomochain/common" + "github.com/tomochain/tomochain/common/math" + "github.com/tomochain/tomochain/crypto" + "github.com/tomochain/tomochain/params" "github.com/tomochain/tomochain/rlp" ) @@ -59,3 +62,128 @@ func TestBlockEncoding(t *testing.T) { t.Errorf("encoded block mismatch:\ngot: %x\nwant: %x", ourBlockEnc, blockEnc) } } + +func TestUncleHash(t *testing.T) { + uncles := make([]*Header, 0) + h := CalcUncleHash(uncles) + exp := common.HexToHash("1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347") + if h != exp { + t.Fatalf("empty uncle hash is wrong, got %x != %x", h, exp) + } +} + +var benchBuffer = bytes.NewBuffer(make([]byte, 0, 32000)) + +func BenchmarkEncodeBlock(b *testing.B) { + block := makeBenchBlock() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + benchBuffer.Reset() + if err := rlp.Encode(benchBuffer, block); err != nil { + b.Fatal(err) + } + } +} + +func makeBenchBlock() *Block { + var ( + key, _ = crypto.GenerateKey() + txs = make([]*Transaction, 70) + receipts = make([]*Receipt, len(txs)) + signer = LatestSigner(params.TestChainConfig) + uncles = make([]*Header, 3) + ) + header := &Header{ + Difficulty: math.BigPow(11, 11), + Number: math.BigPow(2, 9), + GasLimit: 12345678, + GasUsed: 1476322, + Time: new(big.Int).SetUint64(9876543), + Extra: []byte("coolest block on chain"), + } + for i := range txs { + amount := math.BigPow(2, int64(i)) + price := big.NewInt(300000) + data := make([]byte, 100) + tx := NewTransaction(uint64(i), common.Address{}, amount, 123457, price, data) + signedTx, err := SignTx(tx, signer, key) + if err != nil { + panic(err) + } + txs[i] = signedTx + receipts[i] = NewReceipt(make([]byte, 32), false, tx.Gas()) + } + for i := range uncles { + uncles[i] = &Header{ + Difficulty: math.BigPow(11, 11), + Number: math.BigPow(2, 9), + GasLimit: 12345678, + GasUsed: 1476322, + Time: new(big.Int).SetUint64(9876543), + Extra: []byte("benchmark uncle"), + } + } + return NewBlock(header, txs, uncles, receipts) +} + +func TestRlpDecodeParentHash(t *testing.T) { + // A minimum one + want := common.HexToHash("0x112233445566778899001122334455667788990011223344556677889900aabb") + if rlpData, err := rlp.EncodeToBytes(&Header{ParentHash: want}); err != nil { + t.Fatal(err) + } else { + if have := HeaderParentHashFromRLP(rlpData); have != want { + t.Fatalf("have %x, want %x", have, want) + } + } + // And a maximum one + // | Difficulty | dynamic| *big.Int | 0x5ad3c2c71bbff854908 (current mainnet TD: 76 bits) | + // | Number | dynamic| *big.Int | 64 bits | + // | Extra | dynamic| []byte | 65+32 byte (clique) | + // | BaseFee | dynamic| *big.Int | 64 bits | + mainnetTd := new(big.Int) + mainnetTd.SetString("5ad3c2c71bbff854908", 16) + if rlpData, err := rlp.EncodeToBytes(&Header{ + ParentHash: want, + Difficulty: mainnetTd, + Number: new(big.Int).SetUint64(math.MaxUint64), + Extra: make([]byte, 65+32), + BaseFee: new(big.Int).SetUint64(math.MaxUint64), + }); err != nil { + t.Fatal(err) + } else { + if have := HeaderParentHashFromRLP(rlpData); have != want { + t.Fatalf("have %x, want %x", have, want) + } + } + // Also test a very very large header. + { + // The rlp-encoding of the header belowCauses _total_ length of 65540, + // which is the first to blow the fast-path. + h := &Header{ + ParentHash: want, + Extra: make([]byte, 65041), + } + if rlpData, err := rlp.EncodeToBytes(h); err != nil { + t.Fatal(err) + } else { + if have := HeaderParentHashFromRLP(rlpData); have != want { + t.Fatalf("have %x, want %x", have, want) + } + } + } + { + // Test some invalid erroneous stuff + for i, rlpData := range [][]byte{ + nil, + common.FromHex("0x"), + common.FromHex("0x01"), + common.FromHex("0x3031323334"), + } { + if have, want := HeaderParentHashFromRLP(rlpData), (common.Hash{}); have != want { + t.Fatalf("invalid %d: have %x, want %x", i, have, want) + } + } + } +} diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index 75ff32bb23..9f469aeb89 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -179,7 +179,7 @@ func TestPrestateTracerCreate2(t *testing.T) { } evm := vm.NewEVM(context, statedb, nil, params.MainnetChainConfig, vm.Config{Debug: true, Tracer: tracer}) - msg, err := core.TransactionToMessage(signer, nil, nil) + msg, err := core.TransactionToMessage(tx, signer, nil, nil) if err != nil { t.Fatalf("failed to prepare transaction for tracing: %v", err) } @@ -254,7 +254,7 @@ func TestCallTracer(t *testing.T) { } evm := vm.NewEVM(context, statedb, nil, test.Genesis.Config, vm.Config{Debug: true, Tracer: tracer}) - msg, err := core.TransactionToMessage(signer, nil, common.Big0) + msg, err := core.TransactionToMessage(tx, signer, nil, common.Big0) if err != nil { t.Fatalf("failed to prepare transaction for tracing: %v", err) } From e5ee2d07f594e93cd42c71f7c0a8a84f839187c1 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Mon, 26 Jun 2023 16:26:53 +0700 Subject: [PATCH 22/82] Fix storage size of transactions --- core/tx_pool.go | 2 +- core/types/transaction.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/tx_pool.go b/core/tx_pool.go index a1028ccaa3..11bbd6badb 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -19,7 +19,6 @@ package core import ( "errors" "fmt" - "github.com/tomochain/tomochain/consensus" "math" "math/big" "sort" @@ -27,6 +26,7 @@ import ( "time" "github.com/tomochain/tomochain/common" + "github.com/tomochain/tomochain/consensus" "github.com/tomochain/tomochain/core/state" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/event" diff --git a/core/types/transaction.go b/core/types/transaction.go index c06bf08914..4ed3bf614d 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -295,7 +295,7 @@ func (tx *Transaction) setDecoded(inner TxData, size uint64) { tx.inner = inner tx.time = time.Now() if size > 0 { - tx.size.Store(size) + tx.size.Store(common.StorageSize(size)) } } From 0ea51ca59b7e2a6100f1182a28c358617a977c9f Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Mon, 26 Jun 2023 16:33:41 +0700 Subject: [PATCH 23/82] Add London signer to MakeSigner function --- core/types/transaction_signing.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index 17ccdcf8ca..c15516759a 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -42,6 +42,8 @@ type sigCache struct { func MakeSigner(config *params.ChainConfig, blockNumber *big.Int) Signer { var signer Signer switch { + case config.IsLondon(blockNumber): + signer = NewLondonSigner(config.ChainId) case config.IsEIP155(blockNumber): signer = NewEIP155Signer(config.ChainId) case config.IsHomestead(blockNumber): From 83ff4faf0ab965c05d700f86bd3ec409dacb036e Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Mon, 26 Jun 2023 17:46:57 +0700 Subject: [PATCH 24/82] Fix balanceToFee --- internal/ethapi/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index d8710911dc..0ecea18e4a 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1036,7 +1036,7 @@ func (s *PublicBlockChainAPI) doCall(ctx context.Context, args TransactionArgs, if gasPrice.Sign() == 0 { gasPrice = new(big.Int).SetUint64(defaultGasPrice) } - balanceTokenFee, _ := big.NewInt(0).SetString(args.Gas.String(), 10) + balanceTokenFee, _ := big.NewInt(0).SetString(args.Gas.String(), 16) balanceTokenFee = balanceTokenFee.Mul(balanceTokenFee, gasPrice) // Create new call message From 529a6f4df27a99970a9fac92e5f57bbcf99a17b7 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Tue, 27 Jun 2023 10:40:41 +0700 Subject: [PATCH 25/82] Add support for EIP-2718 in abi/bind package --- accounts/abi/bind/backend.go | 3 + accounts/abi/bind/backends/simulated.go | 6 ++ accounts/abi/bind/base.go | 134 +++++++++++++++++++++++- cmd/puppeth/wizard_genesis.go | 1 + 4 files changed, 141 insertions(+), 3 deletions(-) diff --git a/accounts/abi/bind/backend.go b/accounts/abi/bind/backend.go index d6862549f5..ee246d007e 100644 --- a/accounts/abi/bind/backend.go +++ b/accounts/abi/bind/backend.go @@ -74,6 +74,9 @@ type ContractTransactor interface { // SuggestGasPrice retrieves the currently suggested gas price to allow a timely // execution of a transaction. SuggestGasPrice(ctx context.Context) (*big.Int, error) + // SuggestGasTipCap retrieves the currently suggested 1559 priority fee to allow + // a timely execution of a transaction. + SuggestGasTipCap(ctx context.Context) (*big.Int, error) // EstimateGas tries to estimate the gas needed to execute a specific // transaction based on the current pending state of the backend blockchain. // There is no guarantee that this is the true gas limit requirement as other diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 3cd1737c33..fcb82a567c 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -271,6 +271,12 @@ func (b *SimulatedBackend) SuggestGasPrice(ctx context.Context) (*big.Int, error return big.NewInt(1), nil } +// SuggestGasTipCap implements ContractTransactor.SuggestGasTipCap. Since the simulated +// chain doesn't have miners, we just return a gas tip of 1 for any call. +func (b *SimulatedBackend) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { + return big.NewInt(1), nil +} + // EstimateGas executes the requested code against the currently pending block/state and // returns the used amount of gas. func (b *SimulatedBackend) EstimateGas(ctx context.Context, call tomochain.CallMsg) (uint64, error) { diff --git a/accounts/abi/bind/base.go b/accounts/abi/bind/base.go index caf1640496..596951553a 100644 --- a/accounts/abi/bind/base.go +++ b/accounts/abi/bind/base.go @@ -30,6 +30,8 @@ import ( "github.com/tomochain/tomochain/event" ) +const basefeeWiggleMultiplier = 2 + // SignerFn is a signer function callback when a contract requires a method to // sign the transaction before submission. type SignerFn func(types.Signer, common.Address, *types.Transaction) (*types.Transaction, error) @@ -49,9 +51,11 @@ type TransactOpts struct { Nonce *big.Int // Nonce to use for the transaction execution (nil = use pending state) Signer SignerFn // Method to use for signing the transaction (mandatory) - Value *big.Int // Funds to transfer along along the transaction (nil = 0 = no funds) - GasPrice *big.Int // Gas price to use for the transaction execution (nil = gas price oracle) - GasLimit uint64 // Gas limit to set for the transaction execution (0 = estimate) + Value *big.Int // Funds to transfer along along the transaction (nil = 0 = no funds) + GasPrice *big.Int // Gas price to use for the transaction execution (nil = gas price oracle) + GasFeeCap *big.Int // Gas fee cap to use for the 1559 transaction execution (nil = gas price oracle) + GasTipCap *big.Int // Gas priority fee cap to use for the 1559 transaction execution (nil = gas price oracle) + GasLimit uint64 // Gas limit to set for the transaction execution (0 = estimate) Context context.Context // Network context to support cancellation and timeouts (nil = no timeout) } @@ -180,6 +184,130 @@ func (c *BoundContract) Transfer(opts *TransactOpts) (*types.Transaction, error) return c.transact(opts, &c.address, nil) } +func (c *BoundContract) createDynamicTx(opts *TransactOpts, contract *common.Address, input []byte, head *types.Header) (*types.Transaction, error) { + // Normalize value + value := opts.Value + if value == nil { + value = new(big.Int) + } + // Estimate TipCap + gasTipCap := opts.GasTipCap + if gasTipCap == nil { + tip, err := c.transactor.SuggestGasTipCap(ensureContext(opts.Context)) + if err != nil { + return nil, err + } + gasTipCap = tip + } + // Estimate FeeCap + gasFeeCap := opts.GasFeeCap + if gasFeeCap == nil { + gasFeeCap = new(big.Int).Add( + gasTipCap, + new(big.Int).Mul(head.BaseFee, big.NewInt(basefeeWiggleMultiplier)), + ) + } + if gasFeeCap.Cmp(gasTipCap) < 0 { + return nil, fmt.Errorf("maxFeePerGas (%v) < maxPriorityFeePerGas (%v)", gasFeeCap, gasTipCap) + } + // Estimate GasLimit + gasLimit := opts.GasLimit + if opts.GasLimit == 0 { + var err error + gasLimit, err = c.estimateGasLimit(opts, contract, input, nil, gasTipCap, gasFeeCap, value) + if err != nil { + return nil, err + } + } + // create the transaction + nonce, err := c.getNonce(opts) + if err != nil { + return nil, err + } + baseTx := &types.DynamicFeeTx{ + To: contract, + Nonce: nonce, + GasFeeCap: gasFeeCap, + GasTipCap: gasTipCap, + Gas: gasLimit, + Value: value, + Data: input, + } + return types.NewTx(baseTx), nil +} + +func (c *BoundContract) createLegacyTx(opts *TransactOpts, contract *common.Address, input []byte) (*types.Transaction, error) { + if opts.GasFeeCap != nil || opts.GasTipCap != nil { + return nil, errors.New("maxFeePerGas or maxPriorityFeePerGas specified but london is not active yet") + } + // Normalize value + value := opts.Value + if value == nil { + value = new(big.Int) + } + // Estimate GasPrice + gasPrice := opts.GasPrice + if gasPrice == nil { + price, err := c.transactor.SuggestGasPrice(ensureContext(opts.Context)) + if err != nil { + return nil, err + } + gasPrice = price + } + // Estimate GasLimit + gasLimit := opts.GasLimit + if opts.GasLimit == 0 { + var err error + gasLimit, err = c.estimateGasLimit(opts, contract, input, gasPrice, nil, nil, value) + if err != nil { + return nil, err + } + } + // create the transaction + nonce, err := c.getNonce(opts) + if err != nil { + return nil, err + } + baseTx := &types.LegacyTx{ + To: contract, + Nonce: nonce, + GasPrice: gasPrice, + Gas: gasLimit, + Value: value, + Data: input, + } + return types.NewTx(baseTx), nil +} + +func (c *BoundContract) estimateGasLimit(opts *TransactOpts, contract *common.Address, input []byte, gasPrice, gasTipCap, gasFeeCap, value *big.Int) (uint64, error) { + if contract != nil { + // Gas estimation cannot succeed without code for method invocations. + if code, err := c.transactor.PendingCodeAt(ensureContext(opts.Context), c.address); err != nil { + return 0, err + } else if len(code) == 0 { + return 0, ErrNoCode + } + } + msg := tomochain.CallMsg{ + From: opts.From, + To: contract, + GasPrice: gasPrice, + GasTipCap: gasTipCap, + GasFeeCap: gasFeeCap, + Value: value, + Data: input, + } + return c.transactor.EstimateGas(ensureContext(opts.Context), msg) +} + +func (c *BoundContract) getNonce(opts *TransactOpts) (uint64, error) { + if opts.Nonce == nil { + return c.transactor.PendingNonceAt(ensureContext(opts.Context), opts.From) + } else { + return opts.Nonce.Uint64(), nil + } +} + // transact executes an actual transaction invocation, first deriving any missing // authorization fields, and then scheduling the transaction for execution. func (c *BoundContract) transact(opts *TransactOpts, contract *common.Address, input []byte) (*types.Transaction, error) { diff --git a/cmd/puppeth/wizard_genesis.go b/cmd/puppeth/wizard_genesis.go index 1d278662c6..a2ceb56774 100644 --- a/cmd/puppeth/wizard_genesis.go +++ b/cmd/puppeth/wizard_genesis.go @@ -56,6 +56,7 @@ func (w *wizard) makeGenesis() { EIP155Block: big.NewInt(3), EIP158Block: big.NewInt(3), ByzantiumBlock: big.NewInt(4), + LondonBlock: big.NewInt(50), }, } // Figure out which consensus engine to choose From 5e0ffdc2b2c7a665ca6b26e8011d41873e04046c Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Tue, 27 Jun 2023 11:43:43 +0700 Subject: [PATCH 26/82] Modify gas price oracle and API backend interface --- eth/api_backend.go | 12 +- eth/gasprice/gasprice.go | 271 ++++++++++++++++++++++--------------- ethclient/ethclient.go | 10 ++ ethstats/ethstats.go | 5 +- go.mod | 4 +- go.sum | 9 +- internal/ethapi/api.go | 93 +++++++++++-- internal/ethapi/backend.go | 3 +- les/api_backend.go | 13 +- 9 files changed, 290 insertions(+), 130 deletions(-) diff --git a/eth/api_backend.go b/eth/api_backend.go index bc30f7032f..cd1d589545 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -218,6 +218,10 @@ func (b *EthApiBackend) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Sub return b.eth.TxPool().SubscribeTxPreEvent(ch) } +func (b *EthApiBackend) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { + return b.gpo.SuggestTipCap(ctx) +} + func (b *EthApiBackend) Downloader() *downloader.Downloader { return b.eth.Downloader() } @@ -226,10 +230,6 @@ func (b *EthApiBackend) ProtocolVersion() int { return b.eth.EthVersion() } -func (b *EthApiBackend) SuggestPrice(ctx context.Context) (*big.Int, error) { - return b.gpo.SuggestPrice(ctx) -} - func (b *EthApiBackend) ChainDb() ethdb.Database { return b.eth.ChainDb() } @@ -266,6 +266,10 @@ func (b *EthApiBackend) GetEngine() consensus.Engine { return b.eth.engine } +func (b *EthApiBackend) CurrentHeader() *types.Header { + return b.eth.blockchain.CurrentHeader() +} + func (s *EthApiBackend) GetRewardByHash(hash common.Hash) map[string]map[string]map[string]*big.Int { header := s.eth.blockchain.GetHeaderByHash(hash) if header != nil { diff --git a/eth/gasprice/gasprice.go b/eth/gasprice/gasprice.go index 46b4f21005..96e2bdffc4 100644 --- a/eth/gasprice/gasprice.go +++ b/eth/gasprice/gasprice.go @@ -19,177 +19,232 @@ package gasprice import ( "context" "math/big" - "sort" "sync" "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/internal/ethapi" + "github.com/tomochain/tomochain/log" "github.com/tomochain/tomochain/params" "github.com/tomochain/tomochain/rpc" + + "golang.org/x/exp/slices" +) + +var ( + DefaultMaxPrice = big.NewInt(500 * params.Shannon) + DefaultIgnorePrice = big.NewInt(2 * params.Wei) ) -var maxPrice = big.NewInt(500 * params.Shannon) +const sampleNumber = 3 // Number of transactions sampled in a block type Config struct { - Blocks int - Percentile int - Default *big.Int `toml:",omitempty"` + Blocks int + Percentile int + MaxHeaderHistory uint64 + MaxBlockHistory uint64 + Default *big.Int `toml:",omitempty"` + MaxPrice *big.Int `toml:",omitempty"` + IgnorePrice *big.Int `toml:",omitempty"` } // Oracle recommends gas prices based on the content of recent // blocks. Suitable for both light and full clients. type Oracle struct { - backend ethapi.Backend - lastHead common.Hash - lastPrice *big.Int - cacheLock sync.RWMutex - fetchLock sync.Mutex - - checkBlocks, maxEmpty, maxBlocks int - percentile int + backend ethapi.Backend + lastHead common.Hash + lastPrice *big.Int + maxPrice *big.Int + ignorePrice *big.Int + cacheLock sync.RWMutex + fetchLock sync.Mutex + + checkBlocks, percentile int + maxHeaderHistory, maxBlockHistory uint64 } -// NewOracle returns a new oracle. +// NewOracle returns a new gasprice oracle which can recommend suitable +// gasprice for newly created transaction. func NewOracle(backend ethapi.Backend, params Config) *Oracle { blocks := params.Blocks if blocks < 1 { blocks = 1 + log.Warn("Sanitizing invalid gasprice oracle sample blocks", "provided", params.Blocks, "updated", blocks) } percent := params.Percentile if percent < 0 { percent = 0 - } - if percent > 100 { + log.Warn("Sanitizing invalid gasprice oracle sample percentile", "provided", params.Percentile, "updated", percent) + } else if percent > 100 { percent = 100 + log.Warn("Sanitizing invalid gasprice oracle sample percentile", "provided", params.Percentile, "updated", percent) + } + + maxPrice := params.MaxPrice + if maxPrice == nil || maxPrice.Int64() <= 0 { + maxPrice = DefaultMaxPrice + log.Warn("Sanitizing invalid gasprice oracle price cap", "provided", params.MaxPrice, "updated", maxPrice) + } + ignorePrice := params.IgnorePrice + if ignorePrice == nil || ignorePrice.Int64() <= 0 { + ignorePrice = DefaultIgnorePrice + log.Warn("Sanitizing invalid gasprice oracle ignore price", "provided", params.IgnorePrice, "updated", ignorePrice) + } else if ignorePrice.Int64() > 0 { + log.Info("Gasprice oracle is ignoring threshold set", "threshold", ignorePrice) + } + maxHeaderHistory := params.MaxHeaderHistory + if maxHeaderHistory < 1 { + maxHeaderHistory = 1 + log.Warn("Sanitizing invalid gasprice oracle max header history", "provided", params.MaxHeaderHistory, "updated", maxHeaderHistory) } + maxBlockHistory := params.MaxBlockHistory + if maxBlockHistory < 1 { + maxBlockHistory = 1 + log.Warn("Sanitizing invalid gasprice oracle max block history", "provided", params.MaxBlockHistory, "updated", maxBlockHistory) + } + return &Oracle{ - backend: backend, - lastPrice: params.Default, - checkBlocks: blocks, - maxEmpty: blocks / 2, - maxBlocks: blocks * 5, - percentile: percent, + backend: backend, + lastPrice: params.Default, + maxPrice: maxPrice, + ignorePrice: ignorePrice, + checkBlocks: blocks, + percentile: percent, + maxHeaderHistory: maxHeaderHistory, + maxBlockHistory: maxBlockHistory, } } -// SuggestPrice returns the recommended gas price. -func (gpo *Oracle) SuggestPrice(ctx context.Context) (*big.Int, error) { - gpo.cacheLock.RLock() - lastHead := gpo.lastHead - lastPrice := gpo.lastPrice - gpo.cacheLock.RUnlock() - - head, _ := gpo.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber) +// SuggestTipCap returns a tip cap so that newly created transaction can have a +// very high chance to be included in the following blocks. +// +// Note, for legacy transactions and the legacy eth_gasPrice RPC call, it will be +// necessary to add the basefee to the returned number to fall back to the legacy +// behavior. +func (oracle *Oracle) SuggestTipCap(ctx context.Context) (*big.Int, error) { + head, _ := oracle.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber) headHash := head.Hash() + + // If the latest gasprice is still available, return it. + oracle.cacheLock.RLock() + lastHead, lastPrice := oracle.lastHead, oracle.lastPrice + oracle.cacheLock.RUnlock() if headHash == lastHead { - return lastPrice, nil + return new(big.Int).Set(lastPrice), nil } + oracle.fetchLock.Lock() + defer oracle.fetchLock.Unlock() - gpo.fetchLock.Lock() - defer gpo.fetchLock.Unlock() - - // try checking the cache again, maybe the last fetch fetched what we need - gpo.cacheLock.RLock() - lastHead = gpo.lastHead - lastPrice = gpo.lastPrice - gpo.cacheLock.RUnlock() + // Try checking the cache again, maybe the last fetch fetched what we need + oracle.cacheLock.RLock() + lastHead, lastPrice = oracle.lastHead, oracle.lastPrice + oracle.cacheLock.RUnlock() if headHash == lastHead { - return lastPrice, nil + return new(big.Int).Set(lastPrice), nil } - - blockNum := head.Number.Uint64() - ch := make(chan getBlockPricesResult, gpo.checkBlocks) - sent := 0 - exp := 0 - var blockPrices []*big.Int - for sent < gpo.checkBlocks && blockNum > 0 { - go gpo.getBlockPrices(ctx, types.MakeSigner(gpo.backend.ChainConfig(), big.NewInt(int64(blockNum))), blockNum, ch) + var ( + sent, exp int + number = head.Number.Uint64() + result = make(chan results, oracle.checkBlocks) + quit = make(chan struct{}) + results []*big.Int + ) + for sent < oracle.checkBlocks && number > 0 { + go oracle.getBlockValues(ctx, number, sampleNumber, oracle.ignorePrice, result, quit) sent++ exp++ - blockNum-- + number-- } - maxEmpty := gpo.maxEmpty for exp > 0 { - res := <-ch + res := <-result if res.err != nil { - return lastPrice, res.err + close(quit) + return new(big.Int).Set(lastPrice), res.err } exp-- - if res.price != nil { - blockPrices = append(blockPrices, res.price) - continue - } - if maxEmpty > 0 { - maxEmpty-- - continue + // Nothing returned. There are two special cases here: + // - The block is empty + // - All the transactions included are sent by the miner itself. + // In these cases, use the latest calculated price for sampling. + if len(res.values) == 0 { + res.values = []*big.Int{lastPrice} } - if blockNum > 0 && sent < gpo.maxBlocks { - go gpo.getBlockPrices(ctx, types.MakeSigner(gpo.backend.ChainConfig(), big.NewInt(int64(blockNum))), blockNum, ch) + // Besides, in order to collect enough data for sampling, if nothing + // meaningful returned, try to query more blocks. But the maximum + // is 2*checkBlocks. + if len(res.values) == 1 && len(results)+1+exp < oracle.checkBlocks*2 && number > 0 { + go oracle.getBlockValues(ctx, number, sampleNumber, oracle.ignorePrice, result, quit) sent++ exp++ - blockNum-- + number-- } + results = append(results, res.values...) } price := lastPrice - if len(blockPrices) > 0 { - sort.Sort(bigIntArray(blockPrices)) - price = blockPrices[(len(blockPrices)-1)*gpo.percentile/100] - } - if price.Cmp(maxPrice) > 0 { - price = new(big.Int).Set(maxPrice) + if len(results) > 0 { + slices.SortFunc(results, func(a, b *big.Int) bool { return a.Cmp(b) < 0 }) + price = results[(len(results)-1)*oracle.percentile/100] } - - // Check gas price min. - minGasPrice := common.MinGasPrice - if price.Cmp(minGasPrice) < 0 { - price = new(big.Int).Set(minGasPrice) + if price.Cmp(oracle.maxPrice) > 0 { + price = new(big.Int).Set(oracle.maxPrice) } + oracle.cacheLock.Lock() + oracle.lastHead = headHash + oracle.lastPrice = price + oracle.cacheLock.Unlock() - gpo.cacheLock.Lock() - gpo.lastHead = headHash - gpo.lastPrice = price - gpo.cacheLock.Unlock() - return price, nil + return new(big.Int).Set(price), nil } -type getBlockPricesResult struct { - price *big.Int - err error +type results struct { + values []*big.Int + err error } -type transactionsByGasPrice []*types.Transaction - -func (t transactionsByGasPrice) Len() int { return len(t) } -func (t transactionsByGasPrice) Swap(i, j int) { t[i], t[j] = t[j], t[i] } -func (t transactionsByGasPrice) Less(i, j int) bool { return t[i].GasPrice().Cmp(t[j].GasPrice()) < 0 } - -// getBlockPrices calculates the lowest transaction gas price in a given block -// and sends it to the result channel. If the block is empty, price is nil. -func (gpo *Oracle) getBlockPrices(ctx context.Context, signer types.Signer, blockNum uint64, ch chan getBlockPricesResult) { - block, err := gpo.backend.BlockByNumber(ctx, rpc.BlockNumber(blockNum)) +// getBlockValues calculates the lowest transaction gas price in a given block +// and sends it to the result channel. If the block is empty or all transactions +// are sent by the miner itself(it doesn't make any sense to include this kind of +// transaction prices for sampling), nil gasprice is returned. +func (oracle *Oracle) getBlockValues(ctx context.Context, blockNum uint64, limit int, ignoreUnder *big.Int, result chan results, quit chan struct{}) { + block, err := oracle.backend.BlockByNumber(ctx, rpc.BlockNumber(blockNum)) if block == nil { - ch <- getBlockPricesResult{nil, err} + select { + case result <- results{nil, err}: + case <-quit: + } return } - - blockTxs := block.Transactions() - txs := make([]*types.Transaction, len(blockTxs)) - copy(txs, blockTxs) - sort.Sort(transactionsByGasPrice(txs)) - - for _, tx := range txs { + signer := types.MakeSigner(oracle.backend.ChainConfig(), block.Number()) + + // Sort the transaction by effective tip in ascending sort. + txs := block.Transactions() + sortedTxs := make([]*types.Transaction, len(txs)) + copy(sortedTxs, txs) + baseFee := block.BaseFee() + slices.SortFunc(sortedTxs, func(a, b *types.Transaction) bool { + // It's okay to discard the error because a tx would never be + // accepted into a block with an invalid effective tip. + tip1, _ := a.EffectiveGasTip(baseFee) + tip2, _ := b.EffectiveGasTip(baseFee) + return tip1.Cmp(tip2) < 0 + }) + + var prices []*big.Int + for _, tx := range sortedTxs { + tip, _ := tx.EffectiveGasTip(baseFee) + if ignoreUnder != nil && tip.Cmp(ignoreUnder) == -1 { + continue + } sender, err := types.Sender(signer, tx) if err == nil && sender != block.Coinbase() { - ch <- getBlockPricesResult{tx.GasPrice(), nil} - return + prices = append(prices, tip) + if len(prices) >= limit { + break + } } } - ch <- getBlockPricesResult{nil, nil} + select { + case result <- results{prices, nil}: + case <-quit: + } } - -type bigIntArray []*big.Int - -func (s bigIntArray) Len() int { return len(s) } -func (s bigIntArray) Less(i, j int) bool { return s[i].Cmp(s[j]) < 0 } -func (s bigIntArray) Swap(i, j int) { s[i], s[j] = s[j], s[i] } diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index d4da5b6cc8..4f868b4fc0 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -461,6 +461,16 @@ func (ec *Client) SuggestGasPrice(ctx context.Context) (*big.Int, error) { return (*big.Int)(&hex), nil } +// SuggestGasTipCap retrieves the currently suggested gas tip cap after 1559 to +// allow a timely execution of a transaction. +func (ec *Client) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { + var hex hexutil.Big + if err := ec.c.CallContext(ctx, &hex, "eth_maxPriorityFeePerGas"); err != nil { + return nil, err + } + return (*big.Int)(&hex), nil +} + // EstimateGas tries to estimate the gas needed to execute a specific transaction based on // the current pending state of the backend blockchain. There is no guarantee that this is // the true gas limit requirement as other transactions may be added or removed by miners, diff --git a/ethstats/ethstats.go b/ethstats/ethstats.go index 69825c5a10..92647d2321 100644 --- a/ethstats/ethstats.go +++ b/ethstats/ethstats.go @@ -691,8 +691,11 @@ func (s *Service) reportStats(conn *websocket.Conn) error { sync := s.eth.Downloader().Progress() syncing = s.eth.BlockChain().CurrentHeader().Number.Uint64() >= sync.HighestBlock - price, _ := s.eth.ApiBackend.SuggestPrice(context.Background()) + price, _ := s.eth.ApiBackend.SuggestGasTipCap(context.Background()) gasprice = int(price.Uint64()) + if basefee := s.eth.ApiBackend.CurrentHeader().BaseFee; basefee != nil { + gasprice += int(basefee.Uint64()) + } } else { sync := s.les.Downloader().Progress() syncing = s.les.BlockChain().CurrentHeader().Number.Uint64() >= sync.HighestBlock diff --git a/go.mod b/go.mod index e5db078ba3..f400df76ba 100644 --- a/go.mod +++ b/go.mod @@ -39,6 +39,7 @@ require ( github.com/stretchr/testify v1.8.1 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 golang.org/x/crypto v0.1.0 + golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df golang.org/x/net v0.8.0 golang.org/x/sync v0.1.0 golang.org/x/sys v0.7.0 @@ -55,6 +56,7 @@ require ( github.com/dlclark/regexp2 v1.7.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect + github.com/google/go-cmp v0.5.9 // indirect github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect github.com/google/uuid v1.3.0 // indirect github.com/kr/pretty v0.3.1 // indirect @@ -69,7 +71,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 // indirect - golang.org/x/mod v0.9.0 // indirect + golang.org/x/mod v0.11.0 // indirect golang.org/x/term v0.6.0 // indirect golang.org/x/text v0.8.0 // indirect golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect diff --git a/go.sum b/go.sum index c913b65f5a..78511b41d4 100644 --- a/go.sum +++ b/go.sum @@ -94,8 +94,9 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20230207041349-798e818bf904 h1:4/hN5RUoecvl+RmJRE2YxKWtnnQls6rQjjW5oV7qg2U= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= @@ -245,10 +246,12 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= -golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 0ecea18e4a..879d64e9cf 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -79,7 +79,23 @@ func NewPublicEthereumAPI(b Backend) *PublicEthereumAPI { // GasPrice returns a suggestion for a gas price. func (s *PublicEthereumAPI) GasPrice(ctx context.Context) (*big.Int, error) { - return s.b.SuggestPrice(ctx) + tipcap, err := s.b.SuggestGasTipCap(ctx) + if err != nil { + return nil, err + } + if head := s.b.CurrentHeader(); head.BaseFee != nil { + tipcap.Add(tipcap, head.BaseFee) + } + return tipcap, err +} + +// MaxPriorityFeePerGas returns a suggestion for a gas tip cap for dynamic fee transactions. +func (s *PublicEthereumAPI) MaxPriorityFeePerGas(ctx context.Context) (*big.Int, error) { + tipcap, err := s.b.SuggestGasTipCap(ctx) + if err != nil { + return nil, err + } + return tipcap, err } // ProtocolVersion returns the current Ethereum protocol version this node supports @@ -1761,17 +1777,13 @@ func (args *TransactionArgs) data() []byte { // setDefaults is a helper function that fills in default values for unspecified tx fields. func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { + if err := args.setFeeDefaults(ctx, b); err != nil { + return err + } if args.Gas == nil { args.Gas = new(hexutil.Uint64) *(*uint64)(args.Gas) = 90000 } - if args.GasPrice == nil { - price, err := b.SuggestPrice(ctx) - if err != nil { - return err - } - args.GasPrice = (*hexutil.Big)(price) - } if args.Value == nil { args.Value = new(hexutil.Big) } @@ -1810,6 +1822,71 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { return nil } +func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) error { + // If both gasPrice and at least one of the EIP-1559 fee parameters are specified, error. + if args.GasPrice != nil && (args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil) { + return errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") + } + // If the tx has completely specified a fee mechanism, no default is needed. This allows users + // who are not yet synced past London to get defaults for other tx values. See + // https://github.com/ethereum/go-ethereum/pull/23274 for more information. + eip1559ParamsSet := args.MaxFeePerGas != nil && args.MaxPriorityFeePerGas != nil + if (args.GasPrice != nil && !eip1559ParamsSet) || (args.GasPrice == nil && eip1559ParamsSet) { + // Sanity check the EIP-1559 fee parameters if present. + if args.GasPrice == nil && args.MaxFeePerGas.ToInt().Cmp(args.MaxPriorityFeePerGas.ToInt()) < 0 { + return fmt.Errorf("maxFeePerGas (%v) < maxPriorityFeePerGas (%v)", args.MaxFeePerGas, args.MaxPriorityFeePerGas) + } + return nil + } + // Now attempt to fill in default value depending on whether London is active or not. + head := b.CurrentHeader() + if b.ChainConfig().IsLondon(head.Number) { + // London is active, set maxPriorityFeePerGas and maxFeePerGas. + if err := args.setLondonFeeDefaults(ctx, head, b); err != nil { + return err + } + } else { + if args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil { + return errors.New("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active") + } + // London not active, set gas price. + price, err := b.SuggestGasTipCap(ctx) + if err != nil { + return err + } + args.GasPrice = (*hexutil.Big)(price) + } + return nil +} + +// setLondonFeeDefaults fills in reasonable default fee values for unspecified fields. +func (args *TransactionArgs) setLondonFeeDefaults(ctx context.Context, head *types.Header, b Backend) error { + // Set maxPriorityFeePerGas if it is missing. + if args.MaxPriorityFeePerGas == nil { + tip, err := b.SuggestGasTipCap(ctx) + if err != nil { + return err + } + args.MaxPriorityFeePerGas = (*hexutil.Big)(tip) + } + // Set maxFeePerGas if it is missing. + if args.MaxFeePerGas == nil { + // Set the max fee to be 2 times larger than the previous block's base fee. + // The additional slack allows the tx to not become invalidated if the base + // fee is rising. + val := new(big.Int).Add( + args.MaxPriorityFeePerGas.ToInt(), + new(big.Int).Mul(head.BaseFee, big.NewInt(2)), + ) + args.MaxFeePerGas = (*hexutil.Big)(val) + } + // Both EIP-1559 fee parameters are now set; sanity check them. + if args.MaxFeePerGas.ToInt().Cmp(args.MaxPriorityFeePerGas.ToInt()) < 0 { + return fmt.Errorf("maxFeePerGas (%v) < maxPriorityFeePerGas (%v)", args.MaxFeePerGas, args.MaxPriorityFeePerGas) + } + return nil +} + // ToMessage converts the transaction arguments to the Message type used by the // core evm. This method is used in calls and traces that do not require a real // live transaction. diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index 9a197d4e2a..a043524029 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -45,17 +45,18 @@ type Backend interface { // General Ethereum API Downloader() *downloader.Downloader ProtocolVersion() int - SuggestPrice(ctx context.Context) (*big.Int, error) ChainDb() ethdb.Database EventMux() *event.TypeMux AccountManager() *accounts.Manager TomoxService() *tomox.TomoX LendingService() *tomoxlending.Lending + SuggestGasTipCap(ctx context.Context) (*big.Int, error) // BlockChain API SetHead(number uint64) HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) BlockByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Block, error) + CurrentHeader() *types.Header StateAndHeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*state.StateDB, *types.Header, error) GetBlock(ctx context.Context, blockHash common.Hash) (*types.Block, error) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) diff --git a/les/api_backend.go b/les/api_backend.go index d5ba3c2c91..66f759472c 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -175,6 +175,10 @@ func (b *LesApiBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEven return b.eth.blockchain.SubscribeRemovedLogsEvent(ch) } +func (b *LesApiBackend) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { + return b.gpo.SuggestTipCap(ctx) +} + func (b *LesApiBackend) Downloader() *downloader.Downloader { return b.eth.Downloader() } @@ -183,10 +187,6 @@ func (b *LesApiBackend) ProtocolVersion() int { return b.eth.LesVersion() + 10000 } -func (b *LesApiBackend) SuggestPrice(ctx context.Context) (*big.Int, error) { - return b.gpo.SuggestPrice(ctx) -} - func (b *LesApiBackend) ChainDb() ethdb.Database { return b.eth.chainDb } @@ -220,6 +220,11 @@ func (b *LesApiBackend) GetIPCClient() (*ethclient.Client, error) { func (b *LesApiBackend) GetEngine() consensus.Engine { return b.eth.engine } + +func (b *LesApiBackend) CurrentHeader() *types.Header { + return b.eth.blockchain.CurrentHeader() +} + func (s *LesApiBackend) GetRewardByHash(hash common.Hash) map[string]map[string]map[string]*big.Int { header := s.eth.blockchain.GetHeaderByHash(hash) if header != nil { From 6098691c1049c1697c02e14bd253686f984a599a Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Tue, 27 Jun 2023 11:57:43 +0700 Subject: [PATCH 27/82] Fix balanceTokenFee in doCall --- internal/ethapi/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 879d64e9cf..3f587eaa73 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1052,7 +1052,7 @@ func (s *PublicBlockChainAPI) doCall(ctx context.Context, args TransactionArgs, if gasPrice.Sign() == 0 { gasPrice = new(big.Int).SetUint64(defaultGasPrice) } - balanceTokenFee, _ := big.NewInt(0).SetString(args.Gas.String(), 16) + balanceTokenFee := big.NewInt(0).SetUint64(uint64(*args.Gas)) balanceTokenFee = balanceTokenFee.Mul(balanceTokenFee, gasPrice) // Create new call message From 51792238b8ee69eb7164b32edc1d1a74b3bbf69c Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Tue, 27 Jun 2023 13:56:56 +0700 Subject: [PATCH 28/82] Calculate header BaseFee in worker and chain maker --- core/block_validator.go | 1 + core/chain_makers.go | 48 +++++++++++++++++++++++++++++++++++++-- internal/ethapi/api.go | 13 ++++++++++- miner/worker.go | 7 ++++++ params/config.go | 10 ++++++++ params/protocol_params.go | 6 ++++- 6 files changed, 81 insertions(+), 4 deletions(-) diff --git a/core/block_validator.go b/core/block_validator.go index 34fde4cedd..fe8be62118 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -18,6 +18,7 @@ package core import ( "fmt" + "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/consensus" "github.com/tomochain/tomochain/consensus/posv" diff --git a/core/chain_makers.go b/core/chain_makers.go index ac7c311fd2..40f75c1f71 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -18,12 +18,12 @@ package core import ( "fmt" - "github.com/tomochain/tomochain/core/rawdb" "math/big" "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/consensus" "github.com/tomochain/tomochain/consensus/misc" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/state" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/core/vm" @@ -140,6 +140,27 @@ func (b *BlockGen) TxNonce(addr common.Address) uint64 { // AddUncle adds an uncle header to the generated block. func (b *BlockGen) AddUncle(h *types.Header) { + // The uncle will have the same timestamp and auto-generated difficulty + h.Time = b.header.Time + + var parent *types.Block + for i := b.i - 1; i >= 0; i-- { + if b.chain[i].Hash() == h.ParentHash { + parent = b.chain[i] + break + } + } + chainreader := &fakeChainReader{config: b.config} + h.Difficulty = b.engine.CalcDifficulty(chainreader, b.header.Time.Uint64(), parent.Header()) + + // The gas limit and price should be derived from the parent + h.GasLimit = parent.Header().GasLimit + if b.config.IsLondon(h.Number) { + h.BaseFee = misc.CalcBaseFee(b.config, parent.Header()) + if !b.config.IsLondon(parent.Header().Number) { + h.GasLimit = CalcGasLimit(parent) + } + } b.uncles = append(b.uncles, h) } @@ -245,7 +266,7 @@ func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.S time = new(big.Int).Add(parent.Time(), big.NewInt(10)) // block time is fixed at 10 seconds } - return &types.Header{ + header := &types.Header{ Root: state.IntermediateRoot(chain.Config().IsEIP158(parent.Number())), ParentHash: parent.Hash(), Coinbase: parent.Coinbase(), @@ -259,6 +280,13 @@ func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.S Number: new(big.Int).Add(parent.Number(), common.Big1), Time: time, } + if chain.Config().IsLondon(header.Number) { + header.BaseFee = misc.CalcBaseFee(chain.Config(), parent.Header()) + if !chain.Config().IsLondon(parent.Number()) { + header.GasLimit = CalcGasLimit(parent) + } + } + return header } // newCanonical creates a chain database, and injects a deterministic canonical @@ -304,3 +332,19 @@ func makeBlockChain(parent *types.Block, n int, engine consensus.Engine, db ethd }) return blocks } + +type fakeChainReader struct { + config *params.ChainConfig +} + +// Config returns the chain configuration. +func (cr *fakeChainReader) Config() *params.ChainConfig { + return cr.config +} + +func (cr *fakeChainReader) CurrentHeader() *types.Header { return nil } +func (cr *fakeChainReader) GetHeaderByNumber(number uint64) *types.Header { return nil } +func (cr *fakeChainReader) GetHeaderByHash(hash common.Hash) *types.Header { return nil } +func (cr *fakeChainReader) GetHeader(hash common.Hash, number uint64) *types.Header { return nil } +func (cr *fakeChainReader) GetBlock(hash common.Hash, number uint64) *types.Block { return nil } +func (cr *fakeChainReader) GetTd(hash common.Hash, number uint64) *big.Int { return nil } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 3f587eaa73..baa85c79da 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1047,8 +1047,11 @@ func (s *PublicBlockChainAPI) doCall(ctx context.Context, args TransactionArgs, } } } - // Set default gas price if none were set + // Set default gas price if none were set. gasPrice := args.GasPrice.ToInt() + if gasPrice == nil { + gasPrice = args.MaxFeePerGas.ToInt() + } if gasPrice.Sign() == 0 { gasPrice = new(big.Int).SetUint64(defaultGasPrice) } @@ -1124,6 +1127,10 @@ func (s *PublicBlockChainAPI) EstimateGas(ctx context.Context, args TransactionA hi uint64 cap uint64 ) + // Use zero address if sender unspecified. + if args.From == nil { + args.From = new(common.Address) + } // Determine the highest gas limit can be used during the estimation. if args.Gas != nil && uint64(*args.Gas) >= params.TxGas { hi = uint64(*args.Gas) @@ -1294,6 +1301,10 @@ func (s *PublicBlockChainAPI) rpcOutputBlock(b *types.Block, inclTx bool, fullTx "penalties": hexutil.Bytes(head.Penalties), } + if head.BaseFee != nil { + fields["baseFeePerGas"] = (*hexutil.Big)(head.BaseFee) + } + if inclTx { formatTx := func(tx *types.Transaction) (interface{}, error) { return tx.Hash(), nil diff --git a/miner/worker.go b/miner/worker.go index 1452c20a77..c354d78cb7 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -586,6 +586,13 @@ func (self *worker) commitNewWork() { Extra: self.extra, Time: big.NewInt(tstamp), } + // Set baseFee and GasLimit if we are on an EIP-1559 chain + if self.config.IsLondon(header.Number) { + header.BaseFee = misc.CalcBaseFee(self.config, parent.Header()) + if !self.config.IsLondon(parent.Number()) { + header.GasLimit = core.CalcGasLimit(parent) + } + } // Only set the coinbase if we are mining (avoid spurious block rewards) if atomic.LoadInt32(&self.mining) == 1 { header.Coinbase = self.coinbase diff --git a/params/config.go b/params/config.go index 339a9f8aca..e202b9c8e7 100644 --- a/params/config.go +++ b/params/config.go @@ -345,6 +345,16 @@ func (c *ChainConfig) checkCompatible(newcfg *ChainConfig, head *big.Int) *Confi return nil } +// BaseFeeChangeDenominator bounds the amount the base fee can change between blocks. +func (c *ChainConfig) BaseFeeChangeDenominator() uint64 { + return DefaultBaseFeeChangeDenominator +} + +// ElasticityMultiplier bounds the maximum gas limit an EIP-1559 block may have. +func (c *ChainConfig) ElasticityMultiplier() uint64 { + return DefaultElasticityMultiplier +} + // isForkIncompatible returns true if a fork scheduled at s1 cannot be rescheduled to // block s2 because head is already past the fork. func isForkIncompatible(s1, s2, head *big.Int) bool { diff --git a/params/protocol_params.go b/params/protocol_params.go index 60f16a12c7..86c8d88af7 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -142,6 +142,10 @@ const ( // Introduced in Tangerine Whistle (Eip 150) CreateBySelfdestructGas uint64 = 25000 + DefaultBaseFeeChangeDenominator = 8 // Bounds the amount the base fee can change between blocks. + DefaultElasticityMultiplier = 2 // Bounds the maximum gas limit an EIP-1559 block may have. + InitialBaseFee = 1000000000 // Initial base fee for EIP-1559 blocks. + Bn256AddGasByzantium uint64 = 500 // Byzantium gas needed for an elliptic curve addition Bn256AddGasIstanbul uint64 = 150 // Gas needed for an elliptic curve addition Bn256ScalarMulGasByzantium uint64 = 40000 // Byzantium gas needed for an elliptic curve scalar multiplication @@ -150,4 +154,4 @@ const ( Bn256PairingBaseGasIstanbul uint64 = 45000 // Base price for an elliptic curve pairing check Bn256PairingPerPointGasByzantium uint64 = 80000 // Byzantium per-point price for an elliptic curve pairing check Bn256PairingPerPointGasIstanbul uint64 = 34000 // Per-point price for an elliptic curve pairing check -) \ No newline at end of file +) From 4980fcae4113ad99f45a5f4e6ecf72f520929149 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Tue, 27 Jun 2023 13:58:32 +0700 Subject: [PATCH 29/82] CalBaseFee --- consensus/misc/eip1559.go | 112 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 consensus/misc/eip1559.go diff --git a/consensus/misc/eip1559.go b/consensus/misc/eip1559.go new file mode 100644 index 0000000000..813ad97545 --- /dev/null +++ b/consensus/misc/eip1559.go @@ -0,0 +1,112 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package misc + +import ( + "errors" + "fmt" + "math/big" + + "github.com/tomochain/tomochain/common" + "github.com/tomochain/tomochain/common/math" + "github.com/tomochain/tomochain/core/types" + "github.com/tomochain/tomochain/params" +) + +// VerifyEIP1559Header verifies some header attributes which were changed in EIP-1559, +// - gas limit check +// - basefee check +func VerifyEIP1559Header(config *params.ChainConfig, parent, header *types.Header) error { + // Verify that the gas limit remains within allowed bounds + parentGasLimit := parent.GasLimit + if !config.IsLondon(parent.Number) { + parentGasLimit = parent.GasLimit * config.ElasticityMultiplier() + } + if err := VerifyGaslimit(parentGasLimit, header.GasLimit); err != nil { + return err + } + // Verify the header is not malformed + if header.BaseFee == nil { + return errors.New("header is missing baseFee") + } + // Verify the baseFee is correct based on the parent header. + expectedBaseFee := CalcBaseFee(config, parent) + if header.BaseFee.Cmp(expectedBaseFee) != 0 { + return fmt.Errorf("invalid baseFee: have %s, want %s, parentBaseFee %s, parentGasUsed %d", + header.BaseFee, expectedBaseFee, parent.BaseFee, parent.GasUsed) + } + return nil +} + +// CalcBaseFee calculates the basefee of the header. +func CalcBaseFee(config *params.ChainConfig, parent *types.Header) *big.Int { + // If the current block is the first EIP-1559 block, return the InitialBaseFee. + if !config.IsLondon(parent.Number) { + return new(big.Int).SetUint64(params.InitialBaseFee) + } + + parentGasTarget := parent.GasLimit / config.ElasticityMultiplier() + // If the parent gasUsed is the same as the target, the baseFee remains unchanged. + if parent.GasUsed == parentGasTarget { + return new(big.Int).Set(parent.BaseFee) + } + + var ( + num = new(big.Int) + denom = new(big.Int) + ) + + if parent.GasUsed > parentGasTarget { + // If the parent block used more gas than its target, the baseFee should increase. + // max(1, parentBaseFee * gasUsedDelta / parentGasTarget / baseFeeChangeDenominator) + num.SetUint64(parent.GasUsed - parentGasTarget) + num.Mul(num, parent.BaseFee) + num.Div(num, denom.SetUint64(parentGasTarget)) + num.Div(num, denom.SetUint64(config.BaseFeeChangeDenominator())) + baseFeeDelta := math.BigMax(num, common.Big1) + + return num.Add(parent.BaseFee, baseFeeDelta) + } else { + // Otherwise if the parent block used less gas than its target, the baseFee should decrease. + // max(0, parentBaseFee * gasUsedDelta / parentGasTarget / baseFeeChangeDenominator) + num.SetUint64(parentGasTarget - parent.GasUsed) + num.Mul(num, parent.BaseFee) + num.Div(num, denom.SetUint64(parentGasTarget)) + num.Div(num, denom.SetUint64(config.BaseFeeChangeDenominator())) + baseFee := num.Sub(parent.BaseFee, num) + + return math.BigMax(baseFee, common.Big0) + } +} + +// VerifyGaslimit verifies the header gas limit according increase/decrease +// in relation to the parent gas limit. +func VerifyGaslimit(parentGasLimit, headerGasLimit uint64) error { + // Verify that the gas limit remains within allowed bounds + diff := int64(parentGasLimit) - int64(headerGasLimit) + if diff < 0 { + diff *= -1 + } + limit := parentGasLimit / params.GasLimitBoundDivisor + if uint64(diff) >= limit { + return fmt.Errorf("invalid gas limit: have %d, want %d +-= %d", headerGasLimit, parentGasLimit, limit-1) + } + if headerGasLimit < params.MinGasLimit { + return errors.New("invalid gas limit below 5000") + } + return nil +} From 989bb51fadb3460562010830834de2339602a3cb Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 28 Jun 2023 10:40:07 +0700 Subject: [PATCH 30/82] Modify puppeth tool to adapt London hardfork --- cmd/puppeth/genesis.go | 2 ++ core/tx_pool.go | 2 +- params/config.go | 11 ++++++----- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cmd/puppeth/genesis.go b/cmd/puppeth/genesis.go index ebca4a082e..d9e0d65d95 100644 --- a/cmd/puppeth/genesis.go +++ b/cmd/puppeth/genesis.go @@ -39,6 +39,7 @@ type cppEthereumGenesisSpec struct { EIP158ForkBlock hexutil.Uint64 `json:"EIP158ForkBlock"` ByzantiumForkBlock hexutil.Uint64 `json:"byzantiumForkBlock"` ConstantinopleForkBlock hexutil.Uint64 `json:"constantinopleForkBlock"` + LondonForkBlock hexutil.Uint64 `json:"londonForkBlock"` NetworkID hexutil.Uint64 `json:"networkID"` ChainID hexutil.Uint64 `json:"chainID"` MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"` @@ -102,6 +103,7 @@ func newCppEthereumGenesisSpec(network string, genesis *core.Genesis) (*cppEther spec.Params.EIP158ForkBlock = (hexutil.Uint64)(genesis.Config.EIP158Block.Uint64()) spec.Params.ByzantiumForkBlock = (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()) spec.Params.ConstantinopleForkBlock = (hexutil.Uint64)(math.MaxUint64) + spec.Params.LondonForkBlock = (hexutil.Uint64)(genesis.Config.LondonBlock.Uint64()) spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainId.Uint64()) spec.Params.ChainID = (hexutil.Uint64)(genesis.Config.ChainId.Uint64()) diff --git a/core/tx_pool.go b/core/tx_pool.go index 11bbd6badb..91ceea4bbe 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -246,7 +246,7 @@ func NewTxPool(config TxPoolConfig, chainconfig *params.ChainConfig, chain block config: config, chainconfig: chainconfig, chain: chain, - signer: types.NewEIP155Signer(chainconfig.ChainId), + signer: types.LatestSigner(chainconfig), pending: make(map[common.Address]*txList), queue: make(map[common.Address]*txList), beats: make(map[common.Address]time.Time), diff --git a/params/config.go b/params/config.go index e202b9c8e7..5c1804e5f2 100644 --- a/params/config.go +++ b/params/config.go @@ -102,16 +102,16 @@ var ( // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, big.NewInt(0), new(EthashConfig), nil, nil} + AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil, nil} // AllPosvProtocolChanges contains every protocol change (EIPs) introduced // and accepted by the Ethereum core developers into the Posv consensus. // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllPosvProtocolChanges = &ChainConfig{big.NewInt(89), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, big.NewInt(0), nil, nil, &PosvConfig{Period: 0, Epoch: 30000}} - AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, big.NewInt(0), nil, &CliqueConfig{Period: 0, Epoch: 30000}, nil} - TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, big.NewInt(0), new(EthashConfig), nil, nil} + AllPosvProtocolChanges = &ChainConfig{big.NewInt(89), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, &PosvConfig{Period: 0, Epoch: 30000}} + AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}, nil} + TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil, nil} TestRules = TestChainConfig.Rules(new(big.Int)) ) @@ -190,7 +190,7 @@ func (c *ChainConfig) String() string { default: engine = "unknown" } - return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Engine: %v}", + return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v London: %v Engine: %v}", c.ChainId, c.HomesteadBlock, c.DAOForkBlock, @@ -200,6 +200,7 @@ func (c *ChainConfig) String() string { c.EIP158Block, c.ByzantiumBlock, c.ConstantinopleBlock, + c.LondonBlock, engine, ) } From 314a96593572716282f5ee063c855516648d239d Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 28 Jun 2023 12:44:28 +0700 Subject: [PATCH 31/82] Let sendRawTransaction API accept new tx types --- internal/ethapi/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index baa85c79da..7df99ab113 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -2109,7 +2109,7 @@ func (s *PublicTransactionPoolAPI) SendTransaction(ctx context.Context, args Tra // The sender is responsible for signing the transaction and using the correct nonce. func (s *PublicTransactionPoolAPI) SendRawTransaction(ctx context.Context, encodedTx hexutil.Bytes) (common.Hash, error) { tx := new(types.Transaction) - if err := rlp.DecodeBytes(encodedTx, tx); err != nil { + if err := tx.UnmarshalBinary(encodedTx); err != nil { return common.Hash{}, err } return submitTransaction(ctx, s.b, tx) From 3c89cb10f114ae606403d0e7e8f3caf277c79316 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 28 Jun 2023 15:24:44 +0700 Subject: [PATCH 32/82] Implement BASEFEE opcode and include it in EVM context --- accounts/abi/bind/backends/simulated.go | 2 +- core/error.go | 16 ++++++++ core/evm.go | 9 ++++- core/state_transition.go | 41 +++++++++++++++++++ core/vm/eips.go | 21 ++++++++++ core/vm/evm.go | 52 ++++++++++++------------- core/vm/instructions.go | 24 ++++++++---- core/vm/instructions_test.go | 22 +++++------ core/vm/interpreter.go | 3 ++ core/vm/jump_table.go | 9 +++++ core/vm/opcodes.go | 3 ++ core/vm/runtime/env.go | 1 + core/vm/runtime/runtime.go | 7 +++- params/config.go | 2 + 14 files changed, 164 insertions(+), 48 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index fcb82a567c..feb932e930 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -366,7 +366,7 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call tomochain.Call evmContext := core.NewEVMContext(msg, block.Header(), b.blockchain, nil) // Create a new environment which holds all relevant information // about the transaction and calling mechanisms. - vmenv := vm.NewEVM(evmContext, statedb, nil, b.config, vm.Config{}) + vmenv := vm.NewEVM(evmContext, statedb, nil, b.config, vm.Config{NoBaseFee: true}) gaspool := new(core.GasPool).AddGas(math.MaxUint64) owner := common.Address{} return core.NewStateTransition(vmenv, msg, gaspool).TransitionDb(owner) diff --git a/core/error.go b/core/error.go index 32fa6cee66..a2737c4f29 100644 --- a/core/error.go +++ b/core/error.go @@ -47,6 +47,22 @@ var ( ErrStopPreparingBlock = errors.New("stop calculating a block not verified by M2") + // ErrTipAboveFeeCap is a sanity error to ensure no one is able to specify a + // transaction with a tip higher than the total fee cap. + ErrTipAboveFeeCap = errors.New("max priority fee per gas higher than max fee per gas") + + // ErrTipVeryHigh is a sanity error to avoid extremely big numbers specified + // in the tip field. + ErrTipVeryHigh = errors.New("max priority fee per gas higher than 2^256-1") + + // ErrFeeCapVeryHigh is a sanity error to avoid extremely big numbers specified + // in the fee cap field. + ErrFeeCapVeryHigh = errors.New("max fee per gas higher than 2^256-1") + + // ErrFeeCapTooLow is returned if the transaction fee cap is less than the + // base fee of the block. + ErrFeeCapTooLow = errors.New("max fee per gas less than block base fee") + // ErrSenderNoEOA is returned if the sender of a transaction is a contract. ErrSenderNoEOA = errors.New("sender not an eoa") ) diff --git a/core/evm.go b/core/evm.go index f3ac62a73f..813da809d7 100644 --- a/core/evm.go +++ b/core/evm.go @@ -28,12 +28,18 @@ import ( // NewEVMContext creates a new context for use in the EVM. func NewEVMContext(msg *Message, header *types.Header, chain consensus.ChainContext, author *common.Address) vm.Context { // If we don't have an explicit author (i.e. not mining), extract from the header - var beneficiary common.Address + var ( + beneficiary common.Address + baseFee *big.Int + ) if author == nil { beneficiary, _ = chain.Engine().Author(header) // Ignore error, we're past header validation } else { beneficiary = *author } + if header.BaseFee != nil { + baseFee = new(big.Int).Set(header.BaseFee) + } return vm.Context{ CanTransfer: CanTransfer, Transfer: Transfer, @@ -45,6 +51,7 @@ func NewEVMContext(msg *Message, header *types.Header, chain consensus.ChainCont Difficulty: new(big.Int).Set(header.Difficulty), GasLimit: header.GasLimit, GasPrice: new(big.Int).Set(msg.GasPrice), + BaseFee: baseFee, } } diff --git a/core/state_transition.go b/core/state_transition.go index f64492c45a..8eef0a1b27 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -253,6 +253,32 @@ func (st *StateTransition) preCheck() error { msg.From.Hex(), codeHash) } } + + // Make sure that transaction gasFeeCap is greater than the baseFee (post london) + if st.evm.ChainConfig().IsLondon(st.evm.Context.BlockNumber) { + // Skip the checks if gas fields are zero and baseFee was explicitly disabled (eth_call) + if !st.evm.Config.NoBaseFee || msg.GasFeeCap.BitLen() > 0 || msg.GasTipCap.BitLen() > 0 { + if l := msg.GasFeeCap.BitLen(); l > 256 { + return fmt.Errorf("%w: address %v, maxFeePerGas bit length: %d", ErrFeeCapVeryHigh, + msg.From.Hex(), l) + } + if l := msg.GasTipCap.BitLen(); l > 256 { + return fmt.Errorf("%w: address %v, maxPriorityFeePerGas bit length: %d", ErrTipVeryHigh, + msg.From.Hex(), l) + } + if msg.GasFeeCap.Cmp(msg.GasTipCap) < 0 { + return fmt.Errorf("%w: address %v, maxPriorityFeePerGas: %s, maxFeePerGas: %s", ErrTipAboveFeeCap, + msg.From.Hex(), msg.GasTipCap, msg.GasFeeCap) + } + // This will panic if baseFee is nil, but basefee presence is verified + // as part of header validation. + if msg.GasFeeCap.Cmp(st.evm.Context.BaseFee) < 0 { + return fmt.Errorf("%w: address %v, maxFeePerGas: %s baseFee: %s", ErrFeeCapTooLow, + msg.From.Hex(), msg.GasFeeCap, st.evm.Context.BaseFee) + } + } + } + return st.buyGas() } @@ -284,6 +310,7 @@ func (st *StateTransition) TransitionDb(owner common.Address) (ret []byte, usedG // not assigned to err, except for insufficient balance // error. vmerr error + rules = st.evm.ChainConfig().Rules(st.evm.Context.BlockNumber) ) // for debugging purpose // TODO: clean it after fixing the issue https://github.com/tomochain/tomochain/issues/401 @@ -309,6 +336,20 @@ func (st *StateTransition) TransitionDb(owner common.Address) (ret []byte, usedG } } st.refundGas() + effectiveTip := msg.GasPrice + if rules.IsLondon { + effectiveTip = cmath.BigMin(msg.GasTipCap, new(big.Int).Sub(msg.GasFeeCap, st.evm.Context.BaseFee)) + } + + if st.evm.Config.NoBaseFee && msg.GasFeeCap.Sign() == 0 && msg.GasTipCap.Sign() == 0 { + // Skip fee payment when NoBaseFee is set and the fee fields + // are 0. This avoids a negative effectiveTip being applied to + // the coinbase when simulating calls. + } else { + fee := new(big.Int).SetUint64(st.gasUsed()) + fee.Mul(fee, effectiveTip) + st.state.AddBalance(st.evm.Context.Coinbase, fee) + } if st.evm.BlockNumber.Cmp(common.TIPTRC21Fee) > 0 { if (owner != common.Address{}) { diff --git a/core/vm/eips.go b/core/vm/eips.go index be00512c84..46d2c0727b 100644 --- a/core/vm/eips.go +++ b/core/vm/eips.go @@ -18,6 +18,7 @@ package vm import ( "fmt" + "github.com/tomochain/tomochain/params" ) @@ -26,6 +27,8 @@ import ( // defined jump tables are not polluted. func EnableEIP(eipNum int, jt *JumpTable) error { switch eipNum { + case 3198: + enable3198(jt) case 2200: enable2200(jt) case 1884: @@ -90,3 +93,21 @@ func enable2200(jt *JumpTable) { jt[SLOAD].constantGas = params.SloadGasEIP2200 jt[SSTORE].dynamicGas = gasSStoreEIP2200 } + +// enable3198 applies EIP-3198 (BASEFEE Opcode) +// - Adds an opcode that returns the current block's base fee. +func enable3198(jt *JumpTable) { + // New opcode + jt[BASEFEE] = operation{ + execute: opBaseFee, + constantGas: GasQuickStep, + minStack: minStack(0, 1), + maxStack: maxStack(0, 1), + } +} + +// opBaseFee implements BASEFEE opcode +func opBaseFee(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { + callContext.stack.push(interpreter.evm.Context.BaseFee) + return nil, nil +} diff --git a/core/vm/evm.go b/core/vm/evm.go index 4bd7ff77a1..88654455ec 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -17,15 +17,15 @@ package vm import ( - "github.com/tomochain/tomochain/tomox/tradingstate" "errors" - "github.com/tomochain/tomochain/params" "math/big" "sync/atomic" "time" "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/crypto" + "github.com/tomochain/tomochain/params" + "github.com/tomochain/tomochain/tomox/tradingstate" ) // emptyCodeHash is used by create to ensure deployment is disallowed to already @@ -104,6 +104,7 @@ type Context struct { BlockNumber *big.Int // Provides information for NUMBER Time *big.Int // Provides information for TIME Difficulty *big.Int // Provides information for DIFFICULTY + BaseFee *big.Int // Provides information for BASEFEE } // EVM is the Ethereum Virtual Machine base object and provides @@ -132,7 +133,7 @@ type EVM struct { chainRules params.Rules // virtual machine configuration options used to initialise the // evm. - vmConfig Config + Config Config // global (to this context) ethereum virtual machine // used throughout the execution of the tx. interpreters []Interpreter @@ -150,16 +151,16 @@ type EVM struct { // only ever be used *once*. func NewEVM(ctx Context, statedb StateDB, tradingStateDB *tradingstate.TradingStateDB, chainConfig *params.ChainConfig, vmConfig Config) *EVM { evm := &EVM{ - Context: ctx, - StateDB: statedb, + Context: ctx, + StateDB: statedb, tradingStateDB: tradingStateDB, - vmConfig: vmConfig, - chainConfig: chainConfig, - chainRules: chainConfig.Rules(ctx.BlockNumber), - interpreters: make([]Interpreter, 0, 1), + Config: vmConfig, + chainConfig: chainConfig, + chainRules: chainConfig.Rules(ctx.BlockNumber), + interpreters: make([]Interpreter, 0, 1), } - // vmConfig.EVMInterpreter will be used by EVM-C, it won't be checked here + // Config.EVMInterpreter will be used by EVM-C, it won't be checked here // as we always want to have the built-in EVM as the failover option. evm.interpreters = append(evm.interpreters, NewEVMInterpreter(evm, vmConfig)) evm.interpreter = evm.interpreters[0] @@ -188,7 +189,7 @@ func (evm *EVM) Interpreter() Interpreter { // the necessary steps to create accounts and reverses the state in case of an // execution error or failed value transfer. func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas uint64, value *big.Int) (ret []byte, leftOverGas uint64, err error) { - if evm.vmConfig.NoRecursion && evm.depth > 0 { + if evm.Config.NoRecursion && evm.depth > 0 { return nil, gas, nil } // Fail if we're trying to execute above the call depth limit @@ -215,9 +216,9 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas } if precompiles[addr] == nil && evm.chainRules.IsEIP158 && value.Sign() == 0 { // Calling a non existing account, don't do anything, but ping the tracer - if evm.vmConfig.Debug && evm.depth == 0 { - evm.vmConfig.Tracer.CaptureStart(caller.Address(), addr, false, input, gas, value) - evm.vmConfig.Tracer.CaptureEnd(ret, 0, 0, nil) + if evm.Config.Debug && evm.depth == 0 { + evm.Config.Tracer.CaptureStart(caller.Address(), addr, false, input, gas, value) + evm.Config.Tracer.CaptureEnd(ret, 0, 0, nil) } return nil, gas, nil } @@ -233,11 +234,11 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas start := time.Now() // Capture the tracer start/end events in debug mode - if evm.vmConfig.Debug && evm.depth == 0 { - evm.vmConfig.Tracer.CaptureStart(caller.Address(), addr, false, input, gas, value) + if evm.Config.Debug && evm.depth == 0 { + evm.Config.Tracer.CaptureStart(caller.Address(), addr, false, input, gas, value) defer func() { // Lazy evaluation of the parameters - evm.vmConfig.Tracer.CaptureEnd(ret, gas-contract.Gas, time.Since(start), err) + evm.Config.Tracer.CaptureEnd(ret, gas-contract.Gas, time.Since(start), err) }() } ret, err = run(evm, contract, input, false) @@ -262,7 +263,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas // CallCode differs from Call in the sense that it executes the given address' // code with the caller as context. func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, gas uint64, value *big.Int) (ret []byte, leftOverGas uint64, err error) { - if evm.vmConfig.NoRecursion && evm.depth > 0 { + if evm.Config.NoRecursion && evm.depth > 0 { return nil, gas, nil } // Fail if we're trying to execute above the call depth limit @@ -301,7 +302,7 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, // DelegateCall differs from CallCode in the sense that it executes the given address' // code with the caller as context and the caller is set to the caller of the caller. func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []byte, gas uint64) (ret []byte, leftOverGas uint64, err error) { - if evm.vmConfig.NoRecursion && evm.depth > 0 { + if evm.Config.NoRecursion && evm.depth > 0 { return nil, gas, nil } // Fail if we're trying to execute above the call depth limit @@ -331,7 +332,7 @@ func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []by // Opcodes that attempt to perform such modifications will result in exceptions // instead of performing the modifications. func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte, gas uint64) (ret []byte, leftOverGas uint64, err error) { - if evm.vmConfig.NoRecursion && evm.depth > 0 { + if evm.Config.NoRecursion && evm.depth > 0 { return nil, gas, nil } // Fail if we're trying to execute above the call depth limit @@ -355,7 +356,6 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte evm.StateDB.AddBalance(addr, bigZero) } - // When an error was returned by the EVM or when setting the creation code // above we revert to the snapshot and consume any gas remaining. Additionally // when we're in Homestead this also counts for code storage gas errors. @@ -412,12 +412,12 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, contract := NewContract(caller, AccountRef(address), value, gas) contract.SetCodeOptionalHash(&address, codeAndHash) - if evm.vmConfig.NoRecursion && evm.depth > 0 { + if evm.Config.NoRecursion && evm.depth > 0 { return nil, address, gas, nil } - if evm.vmConfig.Debug && evm.depth == 0 { - evm.vmConfig.Tracer.CaptureStart(caller.Address(), address, true, codeAndHash.code, gas, value) + if evm.Config.Debug && evm.depth == 0 { + evm.Config.Tracer.CaptureStart(caller.Address(), address, true, codeAndHash.code, gas, value) } start := time.Now() @@ -451,8 +451,8 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, if maxCodeSizeExceeded && err == nil { err = ErrMaxCodeSizeExceeded } - if evm.vmConfig.Debug && evm.depth == 0 { - evm.vmConfig.Tracer.CaptureEnd(ret, gas-contract.Gas, time.Since(start), err) + if evm.Config.Debug && evm.depth == 0 { + evm.Config.Tracer.CaptureEnd(ret, gas-contract.Gas, time.Since(start), err) } return ret, address, contract.Gas, err diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 16f3685852..8150e23e56 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -17,12 +17,13 @@ package vm import ( - "github.com/tomochain/tomochain/params" "math/big" "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/common/math" "github.com/tomochain/tomochain/core/types" + "github.com/tomochain/tomochain/params" + "golang.org/x/crypto/sha3" ) @@ -389,7 +390,7 @@ func opSha3(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]by interpreter.hasher.Read(interpreter.hasherBuf[:]) evm := interpreter.evm - if evm.vmConfig.EnablePreimageRecording { + if evm.Config.EnablePreimageRecording { evm.StateDB.AddPreimage(interpreter.hasherBuf, data) } callContext.stack.push(interpreter.intPool.get().SetBytes(interpreter.hasherBuf[:])) @@ -513,16 +514,21 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx // opExtCodeHash returns the code hash of a specified account. // There are several cases when the function is called, while we can relay everything // to `state.GetCodeHash` function to ensure the correctness. -// (1) Caller tries to get the code hash of a normal contract account, state +// +// (1) Caller tries to get the code hash of a normal contract account, state +// // should return the relative code hash and set it as the result. // -// (2) Caller tries to get the code hash of a non-existent account, state should +// (2) Caller tries to get the code hash of a non-existent account, state should +// // return common.Hash{} and zero will be set as the result. // -// (3) Caller tries to get the code hash for an account without contract code, +// (3) Caller tries to get the code hash for an account without contract code, +// // state should return emptyCodeHash(0xc5d246...) as the result. // -// (4) Caller tries to get the code hash of a precompiled account, the result +// (4) Caller tries to get the code hash of a precompiled account, the result +// // should be zero or emptyCodeHash. // // It is worth noting that in order to avoid unnecessary create and clean, @@ -531,10 +537,12 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx // If the precompile account is not transferred any amount on a private or // customized chain, the return value will be zero. // -// (5) Caller tries to get the code hash for an account which is marked as suicided +// (5) Caller tries to get the code hash for an account which is marked as suicided +// // in the current transaction, the code hash of this account should be returned. // -// (6) Caller tries to get the code hash for an account which is marked as deleted, +// (6) Caller tries to get the code hash for an account which is marked as deleted, +// // this account should be regarded as a non-existent account and zero should be returned. func opExtCodeHash(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { slot := callContext.stack.peek() diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index 1b96aed67b..28e5dcd1c8 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -20,13 +20,13 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/tomochain/tomochain/params" "io/ioutil" "math/big" "testing" "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/crypto" + "github.com/tomochain/tomochain/params" ) type TwoOperandTestcase struct { @@ -92,7 +92,7 @@ func init() { func testTwoOperandOp(t *testing.T, tests []TwoOperandTestcase, opFn executionFunc, name string) { var ( - env = NewEVM(Context{}, nil,nil, params.TestChainConfig, Config{}) + env = NewEVM(Context{}, nil, nil, params.TestChainConfig, Config{}) stack = newstack() pc = uint64(0) evmInterpreter = env.interpreter.(*EVMInterpreter) @@ -211,7 +211,7 @@ func TestSAR(t *testing.T) { // getResult is a convenience function to generate the expected values func getResult(args []*twoOperandParams, opFn executionFunc) []TwoOperandTestcase { var ( - env = NewEVM(Context{}, nil, nil,params.TestChainConfig, Config{}) + env = NewEVM(Context{}, nil, nil, params.TestChainConfig, Config{}) stack = newstack() pc = uint64(0) interpreter = env.interpreter.(*EVMInterpreter) @@ -262,9 +262,9 @@ func TestJsonTestcases(t *testing.T) { func opBenchmark(bench *testing.B, op executionFunc, args ...string) { var ( - env = NewEVM(Context{}, nil,nil, params.TestChainConfig, Config{}) + env = NewEVM(Context{}, nil, nil, params.TestChainConfig, Config{}) stack = newstack() - evmInterpreter = NewEVMInterpreter(env, env.vmConfig) + evmInterpreter = NewEVMInterpreter(env, env.Config) ) env.interpreter = evmInterpreter @@ -497,10 +497,10 @@ func BenchmarkOpIsZero(b *testing.B) { func TestOpMstore(t *testing.T) { var ( - env = NewEVM(Context{}, nil,nil, params.TestChainConfig, Config{}) + env = NewEVM(Context{}, nil, nil, params.TestChainConfig, Config{}) stack = newstack() mem = NewMemory() - evmInterpreter = NewEVMInterpreter(env, env.vmConfig) + evmInterpreter = NewEVMInterpreter(env, env.Config) ) env.interpreter = evmInterpreter @@ -523,10 +523,10 @@ func TestOpMstore(t *testing.T) { func BenchmarkOpMstore(bench *testing.B) { var ( - env = NewEVM(Context{}, nil,nil, params.TestChainConfig, Config{}) + env = NewEVM(Context{}, nil, nil, params.TestChainConfig, Config{}) stack = newstack() mem = NewMemory() - evmInterpreter = NewEVMInterpreter(env, env.vmConfig) + evmInterpreter = NewEVMInterpreter(env, env.Config) ) env.interpreter = evmInterpreter @@ -546,10 +546,10 @@ func BenchmarkOpMstore(bench *testing.B) { func BenchmarkOpSHA3(bench *testing.B) { var ( - env = NewEVM(Context{}, nil,nil, params.TestChainConfig, Config{}) + env = NewEVM(Context{}, nil, nil, params.TestChainConfig, Config{}) stack = newstack() mem = NewMemory() - evmInterpreter = NewEVMInterpreter(env, env.vmConfig) + evmInterpreter = NewEVMInterpreter(env, env.Config) ) env.interpreter = evmInterpreter evmInterpreter.intPool = poolOfIntPools.get() diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index fc5b17a4f3..3c97467150 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -37,6 +37,7 @@ type Config struct { EWASMInterpreter string // External EWASM interpreter options EVMInterpreter string // External EVM interpreter options + NoBaseFee bool // Forces the EIP-1559 baseFee to 0 (needed for 0 price calls) ExtraEips []int // Additional EIPS that are to be enabled } @@ -100,6 +101,8 @@ func NewEVMInterpreter(evm *EVM, cfg Config) *EVMInterpreter { if !cfg.JumpTable[STOP].valid { var jt JumpTable switch { + case evm.chainRules.IsLondon: + jt = londonInstructionSet case evm.chainRules.IsIstanbul: jt = istanbulInstructionSet case evm.chainRules.IsConstantinople: diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index 143bf254f7..166653089f 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -57,11 +57,20 @@ var ( byzantiumInstructionSet = newByzantiumInstructionSet() constantinopleInstructionSet = newConstantinopleInstructionSet() istanbulInstructionSet = newIstanbulInstructionSet() + londonInstructionSet = newLondonInstructionSet() ) // JumpTable contains the EVM opcodes supported at a given fork. type JumpTable [256]operation +// newLondonInstructionSet returns the frontier, homestead, byzantium, +// constantinople, istanbul, petersburg, berlin and london instructions. +func newLondonInstructionSet() JumpTable { + instructionSet := newIstanbulInstructionSet() + enable3198(&instructionSet) // Base fee opcode https://eips.ethereum.org/EIPS/eip-3198 + return instructionSet +} + // newIstanbulInstructionSet returns the frontier, homestead // byzantium, contantinople and petersburg instructions. func newIstanbulInstructionSet() JumpTable { diff --git a/core/vm/opcodes.go b/core/vm/opcodes.go index 322e01d17c..f57c949cfa 100644 --- a/core/vm/opcodes.go +++ b/core/vm/opcodes.go @@ -103,6 +103,7 @@ const ( GASLIMIT CHAINID OpCode = 0x46 SELFBALANCE OpCode = 0x47 + BASEFEE OpCode = 0x48 ) // 0x50 range - 'storage' and execution. @@ -280,6 +281,7 @@ var opCodeToString = map[OpCode]string{ GASLIMIT: "GASLIMIT", CHAINID: "CHAINID", SELFBALANCE: "SELFBALANCE", + BASEFEE: "BASEFEE", // 0x50 range - 'storage' and execution. POP: "POP", @@ -432,6 +434,7 @@ var stringToOp = map[string]OpCode{ "CALLDATASIZE": CALLDATASIZE, "CALLDATACOPY": CALLDATACOPY, "CHAINID": CHAINID, + "BASEFEE": BASEFEE, "DELEGATECALL": DELEGATECALL, "STATICCALL": STATICCALL, "CODESIZE": CODESIZE, diff --git a/core/vm/runtime/env.go b/core/vm/runtime/env.go index b32845b57d..b37e821e16 100644 --- a/core/vm/runtime/env.go +++ b/core/vm/runtime/env.go @@ -33,6 +33,7 @@ func NewEnv(cfg *Config) *vm.EVM { Difficulty: cfg.Difficulty, GasLimit: cfg.GasLimit, GasPrice: cfg.GasPrice, + BaseFee: cfg.BaseFee, } return vm.NewEVM(context, cfg.State, nil, cfg.ChainConfig, cfg.EVMConfig) diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index 683cad1d1c..effb46dfc5 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -17,12 +17,12 @@ package runtime import ( - "github.com/tomochain/tomochain/core/rawdb" "math" "math/big" "time" "github.com/tomochain/tomochain/common" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/state" "github.com/tomochain/tomochain/core/vm" "github.com/tomochain/tomochain/crypto" @@ -43,6 +43,7 @@ type Config struct { Value *big.Int Debug bool EVMConfig vm.Config + BaseFee *big.Int State *state.StateDB GetHashFn func(n uint64) common.Hash @@ -59,6 +60,7 @@ func setDefaults(cfg *Config) { EIP150Block: new(big.Int), EIP155Block: new(big.Int), EIP158Block: new(big.Int), + LondonBlock: new(big.Int), } } @@ -85,6 +87,9 @@ func setDefaults(cfg *Config) { return common.BytesToHash(crypto.Keccak256([]byte(new(big.Int).SetUint64(n).String()))) } } + if cfg.BaseFee == nil { + cfg.BaseFee = big.NewInt(params.InitialBaseFee) + } } // Execute executes the code using the input as call data during the execution. diff --git a/params/config.go b/params/config.go index 5c1804e5f2..41aa1748ad 100644 --- a/params/config.go +++ b/params/config.go @@ -430,6 +430,7 @@ type Rules struct { ChainId *big.Int IsHomestead, IsEIP150, IsEIP155, IsEIP158 bool IsByzantium, IsConstantinople, IsPetersburg, IsIstanbul bool + IsLondon bool } func (c *ChainConfig) Rules(num *big.Int) Rules { @@ -447,5 +448,6 @@ func (c *ChainConfig) Rules(num *big.Int) Rules { IsConstantinople: c.IsConstantinople(num), IsPetersburg: c.IsPetersburg(num), IsIstanbul: c.IsIstanbul(num), + IsLondon: c.IsLondon(num), } } From 83f9500f5e5e55fbbd7cf39d4f162e7304edc8b2 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 28 Jun 2023 15:34:44 +0700 Subject: [PATCH 33/82] Remove legacy callMsg struct in simulated backend --- accounts/abi/bind/backends/simulated.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index feb932e930..cbf09cdcaa 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -480,21 +480,6 @@ func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error { return nil } -// callmsg implements core.Message to allow passing it as a transaction simulator. -type callmsg struct { - tomochain.CallMsg -} - -func (m callmsg) From() common.Address { return m.CallMsg.From } -func (m callmsg) Nonce() uint64 { return 0 } -func (m callmsg) CheckNonce() bool { return false } -func (m callmsg) To() *common.Address { return m.CallMsg.To } -func (m callmsg) GasPrice() *big.Int { return m.CallMsg.GasPrice } -func (m callmsg) Gas() uint64 { return m.CallMsg.Gas } -func (m callmsg) Value() *big.Int { return m.CallMsg.Value } -func (m callmsg) Data() []byte { return m.CallMsg.Data } -func (m callmsg) BalanceTokenFee() *big.Int { return m.CallMsg.BalanceTokenFee } - // filterBackend implements filters.Backend to support filtering for logs without // taking bloom-bits acceleration structures into account. type filterBackend struct { From bf66ac1bfde1f31dd62ccaab12eb5cfc89705d3f Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 28 Jun 2023 16:24:35 +0700 Subject: [PATCH 34/82] Modify RPC APIs to adapt new formats --- consensus/clique/clique.go | 12 +- consensus/ethash/consensus.go | 20 +-- internal/ethapi/api.go | 228 ++++++++++++++++++++-------------- 3 files changed, 157 insertions(+), 103 deletions(-) diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index f63373e17e..7e42aca9f0 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -25,6 +25,8 @@ import ( "sync" "time" + "github.com/tomochain/tomochain/rlp" + lru "github.com/hashicorp/golang-lru" "github.com/tomochain/tomochain/accounts" "github.com/tomochain/tomochain/common" @@ -38,7 +40,6 @@ import ( "github.com/tomochain/tomochain/ethdb" "github.com/tomochain/tomochain/log" "github.com/tomochain/tomochain/params" - "github.com/tomochain/tomochain/rlp" "github.com/tomochain/tomochain/rpc" ) @@ -145,8 +146,7 @@ type SignerFn func(accounts.Account, []byte) ([]byte, error) // or not), which could be abused to produce different hashes for the same header. func sigHash(header *types.Header) (hash common.Hash) { hasher := sha3.NewKeccak256() - - rlp.Encode(hasher, []interface{}{ + enc := []interface{}{ header.ParentHash, header.UncleHash, header.Coinbase, @@ -162,7 +162,11 @@ func sigHash(header *types.Header) (hash common.Hash) { header.Extra[:len(header.Extra)-65], // Yes, this will panic if extra is too short header.MixDigest, header.Nonce, - }) + } + if header.BaseFee != nil { + enc = append(enc, header.BaseFee) + } + rlp.Encode(hasher, enc) hasher.Sum(hash[:0]) return hash } diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 12f63cfde7..deabb20aeb 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -254,16 +254,20 @@ func (ethash *Ethash) verifyHeader(chain consensus.ChainReader, header, parent * return fmt.Errorf("invalid gasUsed: have %d, gasLimit %d", header.GasUsed, header.GasLimit) } - // Verify that the gas limit remains within allowed bounds - diff := int64(parent.GasLimit) - int64(header.GasLimit) - if diff < 0 { - diff *= -1 + // Verify the block's gas usage and (if applicable) verify the base fee. + if !chain.Config().IsLondon(header.Number) { + // Verify BaseFee not present before EIP-1559 fork. + if header.BaseFee != nil { + return fmt.Errorf("invalid baseFee before fork: have %d, expected 'nil'", header.BaseFee) + } + if err := misc.VerifyGaslimit(parent.GasLimit, header.GasLimit); err != nil { + return err + } + } else if err := misc.VerifyEIP1559Header(chain.Config(), parent, header); err != nil { + // Verify the header's EIP-1559 attributes. + return err } - limit := parent.GasLimit / params.GasLimitBoundDivisor - if uint64(diff) >= limit || header.GasLimit < params.MinGasLimit { - return fmt.Errorf("invalid gas limit: have %d, want %d += %d", header.GasLimit, parent.GasLimit, limit) - } // Verify that the block number is parent's +1 if diff := new(big.Int).Sub(header.Number, parent.Number); diff.Cmp(big.NewInt(1)) != 0 { return consensus.ErrInvalidNumber diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 7df99ab113..ef4439241d 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -35,6 +35,7 @@ import ( "github.com/tomochain/tomochain/common/hexutil" "github.com/tomochain/tomochain/common/math" "github.com/tomochain/tomochain/consensus/ethash" + "github.com/tomochain/tomochain/consensus/misc" "github.com/tomochain/tomochain/consensus/posv" contractValidator "github.com/tomochain/tomochain/contracts/validator/contract" "github.com/tomochain/tomochain/core" @@ -144,12 +145,12 @@ func (s *PublicTxPoolAPI) Content() map[string]map[string]map[string]*RPCTransac "queued": make(map[string]map[string]*RPCTransaction), } pending, queue := s.b.TxPoolContent() - + curHeader := s.b.CurrentHeader() // Flatten the pending transactions for account, txs := range pending { dump := make(map[string]*RPCTransaction) for _, tx := range txs { - dump[fmt.Sprintf("%d", tx.Nonce())] = newRPCPendingTransaction(tx) + dump[fmt.Sprintf("%d", tx.Nonce())] = newRPCPendingTransaction(tx, curHeader, s.b.ChainConfig()) } content["pending"][account.Hex()] = dump } @@ -157,7 +158,7 @@ func (s *PublicTxPoolAPI) Content() map[string]map[string]map[string]*RPCTransac for account, txs := range queue { dump := make(map[string]*RPCTransaction) for _, tx := range txs { - dump[fmt.Sprintf("%d", tx.Nonce())] = newRPCPendingTransaction(tx) + dump[fmt.Sprintf("%d", tx.Nonce())] = newRPCPendingTransaction(tx, curHeader, s.b.ChainConfig()) } content["queued"][account.Hex()] = dump } @@ -547,7 +548,7 @@ func (s *PublicBlockChainAPI) GetBalance(ctx context.Context, address common.Add func (s *PublicBlockChainAPI) GetBlockByNumber(ctx context.Context, blockNr rpc.BlockNumber, fullTx bool) (map[string]interface{}, error) { block, err := s.b.BlockByNumber(ctx, blockNr) if block != nil { - response, err := s.rpcOutputBlock(block, true, fullTx, ctx) + response, err := s.rpcMarshalBlock(ctx, block, true, fullTx) if err == nil && blockNr == rpc.PendingBlockNumber { // Pending blocks need to nil out a few fields for _, field := range []string{"hash", "nonce", "miner"} { @@ -564,7 +565,7 @@ func (s *PublicBlockChainAPI) GetBlockByNumber(ctx context.Context, blockNr rpc. func (s *PublicBlockChainAPI) GetBlockByHash(ctx context.Context, blockHash common.Hash, fullTx bool) (map[string]interface{}, error) { block, err := s.b.GetBlock(ctx, blockHash) if block != nil { - return s.rpcOutputBlock(block, true, fullTx, ctx) + return s.rpcMarshalBlock(ctx, block, true, fullTx) } return nil, err } @@ -580,7 +581,7 @@ func (s *PublicBlockChainAPI) GetUncleByBlockNumberAndIndex(ctx context.Context, return nil, nil } block = types.NewBlockWithHeader(uncles[index]) - return s.rpcOutputBlock(block, false, false, ctx) + return s.rpcMarshalBlock(ctx, block, false, false) } return nil, err } @@ -597,7 +598,7 @@ func (s *PublicBlockChainAPI) GetUncleByBlockHashAndIndex(ctx context.Context, b return nil, nil } block = types.NewBlockWithHeader(uncles[index]) - return s.rpcOutputBlock(block, false, false, ctx) + return s.rpcMarshalBlock(ctx, block, false, false) } return nil, err } @@ -1272,14 +1273,11 @@ func FormatLogs(logs []vm.StructLog) []StructLogRes { return formatted } -// rpcOutputBlock converts the given block to the RPC output which depends on fullTx. If inclTx is true transactions are -// returned. When fullTx is true the returned block contains full transaction details, otherwise it will only contain -// transaction hashes. -func (s *PublicBlockChainAPI) rpcOutputBlock(b *types.Block, inclTx bool, fullTx bool, ctx context.Context) (map[string]interface{}, error) { - head := b.Header() // copies the header once - fields := map[string]interface{}{ +// RPCMarshalHeader converts the given header to the RPC output . +func RPCMarshalHeader(head *types.Header) map[string]interface{} { + result := map[string]interface{}{ "number": (*hexutil.Big)(head.Number), - "hash": b.Hash(), + "hash": head.Hash(), "parentHash": head.ParentHash, "nonce": head.Nonce, "mixHash": head.MixDigest, @@ -1288,51 +1286,68 @@ func (s *PublicBlockChainAPI) rpcOutputBlock(b *types.Block, inclTx bool, fullTx "stateRoot": head.Root, "miner": head.Coinbase, "difficulty": (*hexutil.Big)(head.Difficulty), - "totalDifficulty": (*hexutil.Big)(s.b.GetTd(b.Hash())), "extraData": hexutil.Bytes(head.Extra), - "size": hexutil.Uint64(b.Size()), "gasLimit": hexutil.Uint64(head.GasLimit), "gasUsed": hexutil.Uint64(head.GasUsed), - "timestamp": (*hexutil.Big)(head.Time), + "timestamp": head.Time, "transactionsRoot": head.TxHash, "receiptsRoot": head.ReceiptHash, - "validators": hexutil.Bytes(head.Validators), - "validator": hexutil.Bytes(head.Validator), - "penalties": hexutil.Bytes(head.Penalties), } if head.BaseFee != nil { - fields["baseFeePerGas"] = (*hexutil.Big)(head.BaseFee) + result["baseFeePerGas"] = (*hexutil.Big)(head.BaseFee) } + return result +} + +// RPCMarshalBlock converts the given block to the RPC output which depends on fullTx. If inclTx is true transactions are +// returned. When fullTx is true the returned block contains full transaction details, otherwise it will only contain +// transaction hashes. +func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool, config *params.ChainConfig) map[string]interface{} { + fields := RPCMarshalHeader(block.Header()) + fields["size"] = hexutil.Uint64(block.Size()) + if inclTx { - formatTx := func(tx *types.Transaction) (interface{}, error) { - return tx.Hash(), nil + formatTx := func(idx int, tx *types.Transaction) interface{} { + return tx.Hash() } - if fullTx { - formatTx = func(tx *types.Transaction) (interface{}, error) { - return newRPCTransactionFromBlockHash(b, tx.Hash()), nil + formatTx = func(idx int, tx *types.Transaction) interface{} { + return newRPCTransactionFromBlockIndex(block, uint64(idx), config) } } - - txs := b.Transactions() + txs := block.Transactions() transactions := make([]interface{}, len(txs)) - var err error - for i, tx := range b.Transactions() { - if transactions[i], err = formatTx(tx); err != nil { - return nil, err - } + for i, tx := range txs { + transactions[i] = formatTx(i, tx) } fields["transactions"] = transactions } - - uncles := b.Uncles() + uncles := block.Uncles() uncleHashes := make([]common.Hash, len(uncles)) for i, uncle := range uncles { uncleHashes[i] = uncle.Hash() } fields["uncles"] = uncleHashes + return fields +} + +// rpcMarshalHeader uses the generalized output filler, then adds the total difficulty field, which requires +// a `BlockchainAPI`. +func (s *PublicBlockChainAPI) rpcMarshalHeader(ctx context.Context, header *types.Header) map[string]interface{} { + fields := RPCMarshalHeader(header) + fields["totalDifficulty"] = (*hexutil.Big)(s.b.GetTd(header.Hash())) + return fields +} + +// rpcMarshalBlock uses the generalized output filler, then adds the total difficulty field, which requires +// a `BlockchainAPI`. +func (s *PublicBlockChainAPI) rpcMarshalBlock(ctx context.Context, b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { + fields := RPCMarshalBlock(b, inclTx, fullTx, s.b.ChainConfig()) + if inclTx { + fields["totalDifficulty"] = (*hexutil.Big)(s.b.GetTd(b.Hash())) + } return fields, nil } @@ -1486,29 +1501,31 @@ func (s *PublicBlockChainAPI) rpcOutputBlockSigners(b *types.Block, ctx context. // RPCTransaction represents a transaction that will serialize to the RPC representation of a transaction type RPCTransaction struct { - BlockHash common.Hash `json:"blockHash"` - BlockNumber *hexutil.Big `json:"blockNumber"` - From common.Address `json:"from"` - Gas hexutil.Uint64 `json:"gas"` - GasPrice *hexutil.Big `json:"gasPrice"` - Hash common.Hash `json:"hash"` - Input hexutil.Bytes `json:"input"` - Nonce hexutil.Uint64 `json:"nonce"` - To *common.Address `json:"to"` - TransactionIndex hexutil.Uint `json:"transactionIndex"` - Value *hexutil.Big `json:"value"` - V *hexutil.Big `json:"v"` - R *hexutil.Big `json:"r"` - S *hexutil.Big `json:"s"` + BlockHash common.Hash `json:"blockHash"` + BlockNumber *hexutil.Big `json:"blockNumber"` + From common.Address `json:"from"` + Gas hexutil.Uint64 `json:"gas"` + GasPrice *hexutil.Big `json:"gasPrice"` + GasFeeCap *hexutil.Big `json:"maxFeePerGas,omitempty"` + GasTipCap *hexutil.Big `json:"maxPriorityFeePerGas,omitempty"` + Hash common.Hash `json:"hash"` + Input hexutil.Bytes `json:"input"` + Nonce hexutil.Uint64 `json:"nonce"` + To *common.Address `json:"to"` + TransactionIndex hexutil.Uint `json:"transactionIndex"` + Value *hexutil.Big `json:"value"` + Type hexutil.Uint64 `json:"type"` + Accesses *types.AccessList `json:"accessList,omitempty"` + ChainID *hexutil.Big `json:"chainId,omitempty"` + V *hexutil.Big `json:"v"` + R *hexutil.Big `json:"r"` + S *hexutil.Big `json:"s"` } // newRPCTransaction returns a transaction that will serialize to the RPC // representation, with the given location metadata set (if available). -func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber uint64, index uint64) *RPCTransaction { - var signer types.Signer = types.FrontierSigner{} - if tx.Protected() { - signer = types.NewEIP155Signer(tx.ChainId()) - } +func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber uint64, index uint64, baseFee *big.Int, config *params.ChainConfig) *RPCTransaction { + signer := types.MakeSigner(config, new(big.Int).SetUint64(blockNumber)) from, _ := types.Sender(signer, tx) v, r, s := tx.RawSignatureValues() @@ -1530,21 +1547,54 @@ func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber result.BlockNumber = (*hexutil.Big)(new(big.Int).SetUint64(blockNumber)) result.TransactionIndex = hexutil.Uint(index) } + switch tx.Type() { + case types.LegacyTxType: + // if a legacy transaction has an EIP-155 chain id, include it explicitly + if id := tx.ChainId(); id.Sign() != 0 { + result.ChainID = (*hexutil.Big)(id) + } + case types.AccessListTxType: + al := tx.AccessList() + result.Accesses = &al + result.ChainID = (*hexutil.Big)(tx.ChainId()) + case types.DynamicFeeTxType: + al := tx.AccessList() + result.Accesses = &al + result.ChainID = (*hexutil.Big)(tx.ChainId()) + result.GasFeeCap = (*hexutil.Big)(tx.GasFeeCap()) + result.GasTipCap = (*hexutil.Big)(tx.GasTipCap()) + // if the transaction has been mined, compute the effective gas price + if baseFee != nil && blockHash != (common.Hash{}) { + // price = min(tip, gasFeeCap - baseFee) + baseFee + price := math.BigMin(new(big.Int).Add(tx.GasTipCap(), baseFee), tx.GasFeeCap()) + result.GasPrice = (*hexutil.Big)(price) + } else { + result.GasPrice = (*hexutil.Big)(tx.GasFeeCap()) + } + } return result } // newRPCPendingTransaction returns a pending transaction that will serialize to the RPC representation -func newRPCPendingTransaction(tx *types.Transaction) *RPCTransaction { - return newRPCTransaction(tx, common.Hash{}, 0, 0) +func newRPCPendingTransaction(tx *types.Transaction, current *types.Header, config *params.ChainConfig) *RPCTransaction { + var ( + baseFee *big.Int + blockNumber = uint64(0) + ) + if current != nil { + baseFee = misc.CalcBaseFee(config, current) + blockNumber = current.Number.Uint64() + } + return newRPCTransaction(tx, common.Hash{}, blockNumber, 0, baseFee, config) } // newRPCTransactionFromBlockIndex returns a transaction that will serialize to the RPC representation. -func newRPCTransactionFromBlockIndex(b *types.Block, index uint64) *RPCTransaction { +func newRPCTransactionFromBlockIndex(b *types.Block, index uint64, config *params.ChainConfig) *RPCTransaction { txs := b.Transactions() if index >= uint64(len(txs)) { return nil } - return newRPCTransaction(txs[index], b.Hash(), b.NumberU64(), index) + return newRPCTransaction(txs[index], b.Hash(), b.NumberU64(), index, b.BaseFee(), config) } // newRPCRawTransactionFromBlockIndex returns the bytes of a transaction given a block and a transaction index. @@ -1553,24 +1603,15 @@ func newRPCRawTransactionFromBlockIndex(b *types.Block, index uint64) hexutil.By if index >= uint64(len(txs)) { return nil } - blob, _ := rlp.EncodeToBytes(txs[index]) + blob, _ := txs[index].MarshalBinary() return blob } -// newRPCTransactionFromBlockHash returns a transaction that will serialize to the RPC representation. -func newRPCTransactionFromBlockHash(b *types.Block, hash common.Hash) *RPCTransaction { - for idx, tx := range b.Transactions() { - if tx.Hash() == hash { - return newRPCTransactionFromBlockIndex(b, uint64(idx)) - } - } - return nil -} - // PublicTransactionPoolAPI exposes methods for the RPC interface type PublicTransactionPoolAPI struct { b Backend nonceLock *AddrLocker + signer types.Signer } // PublicTransactionPoolAPI exposes methods for the RPC interface @@ -1581,7 +1622,8 @@ type PublicTomoXTransactionPoolAPI struct { // NewPublicTransactionPoolAPI creates a new RPC service with methods specific for the transaction pool. func NewPublicTransactionPoolAPI(b Backend, nonceLock *AddrLocker) *PublicTransactionPoolAPI { - return &PublicTransactionPoolAPI{b, nonceLock} + signer := types.LatestSigner(b.ChainConfig()) + return &PublicTransactionPoolAPI{b, nonceLock, signer} } // NewPublicTransactionPoolAPI creates a new RPC service with methods specific for the transaction pool. @@ -1610,7 +1652,7 @@ func (s *PublicTransactionPoolAPI) GetBlockTransactionCountByHash(ctx context.Co // GetTransactionByBlockNumberAndIndex returns the transaction for the given block number and index. func (s *PublicTransactionPoolAPI) GetTransactionByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) *RPCTransaction { if block, _ := s.b.BlockByNumber(ctx, blockNr); block != nil { - return newRPCTransactionFromBlockIndex(block, uint64(index)) + return newRPCTransactionFromBlockIndex(block, uint64(index), s.b.ChainConfig()) } return nil } @@ -1618,7 +1660,7 @@ func (s *PublicTransactionPoolAPI) GetTransactionByBlockNumberAndIndex(ctx conte // GetTransactionByBlockHashAndIndex returns the transaction for the given block hash and index. func (s *PublicTransactionPoolAPI) GetTransactionByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) *RPCTransaction { if block, _ := s.b.GetBlock(ctx, blockHash); block != nil { - return newRPCTransactionFromBlockIndex(block, uint64(index)) + return newRPCTransactionFromBlockIndex(block, uint64(index), s.b.ChainConfig()) } return nil } @@ -1650,17 +1692,22 @@ func (s *PublicTransactionPoolAPI) GetTransactionCount(ctx context.Context, addr } // GetTransactionByHash returns the transaction for the given hash -func (s *PublicTransactionPoolAPI) GetTransactionByHash(ctx context.Context, hash common.Hash) *RPCTransaction { +func (s *PublicTransactionPoolAPI) GetTransactionByHash(ctx context.Context, hash common.Hash) (*RPCTransaction, error) { // Try to return an already finalized transaction - if tx, blockHash, blockNumber, index := core.GetTransaction(s.b.ChainDb(), hash); tx != nil { - return newRPCTransaction(tx, blockHash, blockNumber, index) + tx, blockHash, blockNumber, index := core.GetTransaction(s.b.ChainDb(), hash) + if tx != nil { + header, err := s.b.HeaderByNumber(ctx, rpc.BlockNumber(blockNumber)) + if err != nil { + return nil, err + } + return newRPCTransaction(tx, blockHash, blockNumber, index, header.BaseFee, s.b.ChainConfig()), nil } // No finalized transaction, try to retrieve it from the pool if tx := s.b.GetPoolTransaction(hash); tx != nil { - return newRPCPendingTransaction(tx) + return newRPCPendingTransaction(tx, s.b.CurrentHeader(), s.b.ChainConfig()), nil } // Transaction unknown, return as such - return nil + return nil, nil } // GetRawTransactionByHash returns the bytes of the transaction for the given hash. @@ -3019,16 +3066,18 @@ func (s *PublicTransactionPoolAPI) PendingTransactions() ([]*RPCTransaction, err if err != nil { return nil, err } - + accounts := make(map[common.Address]struct{}) + for _, wallet := range s.b.AccountManager().Wallets() { + for _, account := range wallet.Accounts() { + accounts[account.Address] = struct{}{} + } + } + curHeader := s.b.CurrentHeader() transactions := make([]*RPCTransaction, 0, len(pending)) for _, tx := range pending { - var signer types.Signer = types.HomesteadSigner{} - if tx.Protected() { - signer = types.NewEIP155Signer(tx.ChainId()) - } - from, _ := types.Sender(signer, tx) - if _, err := s.b.AccountManager().Find(accounts.Account{Address: from}); err == nil { - transactions = append(transactions, newRPCPendingTransaction(tx)) + from, _ := types.Sender(s.signer, tx) + if _, exists := accounts[from]; exists { + transactions = append(transactions, newRPCPendingTransaction(tx, curHeader, s.b.ChainConfig())) } } return transactions, nil @@ -3050,13 +3099,10 @@ func (s *PublicTransactionPoolAPI) Resend(ctx context.Context, sendArgs Transact } for _, p := range pending { - var signer types.Signer = types.HomesteadSigner{} - if p.Protected() { - signer = types.NewEIP155Signer(p.ChainId()) - } - wantSigHash := signer.Hash(matchTx) + pFrom, err := types.Sender(s.signer, p) + wantSigHash := s.signer.Hash(matchTx) - if pFrom, err := types.Sender(signer, p); err == nil && pFrom == sendArgs.from() && signer.Hash(p) == wantSigHash { + if err == nil && pFrom == sendArgs.from() && s.signer.Hash(p) == wantSigHash { // Match. Re-sign and send the transaction. if gasPrice != nil && (*big.Int)(gasPrice).Sign() != 0 { sendArgs.GasPrice = gasPrice From 9ed5d307e380f778a4c7e6ca922d37e0d36167cb Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 28 Jun 2023 16:29:59 +0700 Subject: [PATCH 35/82] Test calBaseFee and gasLimit for new London blocks --- consensus/misc/eip1559_test.go | 126 +++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 consensus/misc/eip1559_test.go diff --git a/consensus/misc/eip1559_test.go b/consensus/misc/eip1559_test.go new file mode 100644 index 0000000000..df5f3e3090 --- /dev/null +++ b/consensus/misc/eip1559_test.go @@ -0,0 +1,126 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package misc + +import ( + "math/big" + "testing" + + "github.com/tomochain/tomochain/common" + "github.com/tomochain/tomochain/core/types" + "github.com/tomochain/tomochain/params" +) + +// copyConfig does a _shallow_ copy of a given config. Safe to set new values, but +// do not use e.g. SetInt() on the numbers. For testing only +func copyConfig(original *params.ChainConfig) *params.ChainConfig { + return ¶ms.ChainConfig{ + ChainId: original.ChainId, + HomesteadBlock: original.HomesteadBlock, + DAOForkBlock: original.DAOForkBlock, + DAOForkSupport: original.DAOForkSupport, + EIP150Block: original.EIP150Block, + EIP155Block: original.EIP155Block, + EIP158Block: original.EIP158Block, + ByzantiumBlock: original.ByzantiumBlock, + ConstantinopleBlock: original.ConstantinopleBlock, + LondonBlock: original.LondonBlock, + Ethash: original.Ethash, + Clique: original.Clique, + } +} + +func config() *params.ChainConfig { + config := copyConfig(params.TestChainConfig) + config.LondonBlock = big.NewInt(5) + return config +} + +// TestBlockGasLimits tests the gasLimit checks for blocks both across +// the EIP-1559 boundary and post-1559 blocks +func TestBlockGasLimits(t *testing.T) { + initial := new(big.Int).SetUint64(params.InitialBaseFee) + + for i, tc := range []struct { + pGasLimit uint64 + pNum int64 + gasLimit uint64 + ok bool + }{ + // Transitions from non-london to london + {10000000, 4, 20000000, true}, // No change + {10000000, 4, 20019530, true}, // Upper limit + {10000000, 4, 20019531, false}, // Upper +1 + {10000000, 4, 19980470, true}, // Lower limit + {10000000, 4, 19980469, false}, // Lower limit -1 + // London to London + {20000000, 5, 20000000, true}, + {20000000, 5, 20019530, true}, // Upper limit + {20000000, 5, 20019531, false}, // Upper limit +1 + {20000000, 5, 19980470, true}, // Lower limit + {20000000, 5, 19980469, false}, // Lower limit -1 + {40000000, 5, 40039061, true}, // Upper limit + {40000000, 5, 40039062, false}, // Upper limit +1 + {40000000, 5, 39960939, true}, // lower limit + {40000000, 5, 39960938, false}, // Lower limit -1 + } { + parent := &types.Header{ + GasUsed: tc.pGasLimit / 2, + GasLimit: tc.pGasLimit, + BaseFee: initial, + Number: big.NewInt(tc.pNum), + } + header := &types.Header{ + GasUsed: tc.gasLimit / 2, + GasLimit: tc.gasLimit, + BaseFee: initial, + Number: big.NewInt(tc.pNum + 1), + } + err := VerifyEIP1559Header(config(), parent, header) + if tc.ok && err != nil { + t.Errorf("test %d: Expected valid header: %s", i, err) + } + if !tc.ok && err == nil { + t.Errorf("test %d: Expected invalid header", i) + } + } +} + +// TestCalcBaseFee assumes all blocks are 1559-blocks +func TestCalcBaseFee(t *testing.T) { + tests := []struct { + parentBaseFee int64 + parentGasLimit uint64 + parentGasUsed uint64 + expectedBaseFee int64 + }{ + {params.InitialBaseFee, 20000000, 10000000, params.InitialBaseFee}, // usage == target + {params.InitialBaseFee, 20000000, 9000000, 987500000}, // usage below target + {params.InitialBaseFee, 20000000, 11000000, 1012500000}, // usage above target + } + for i, test := range tests { + parent := &types.Header{ + Number: common.Big32, + GasLimit: test.parentGasLimit, + GasUsed: test.parentGasUsed, + BaseFee: big.NewInt(test.parentBaseFee), + } + if have, want := CalcBaseFee(config(), parent), big.NewInt(test.expectedBaseFee); have.Cmp(want) != 0 { + t.Errorf("test %d: have %d want %d, ", i, have, want) + } + } +} From ecee2f108e85fef7d2c1d058621a12409b1b40f5 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Fri, 30 Jun 2023 11:08:37 +0700 Subject: [PATCH 36/82] Txpool validate new tx types --- core/tx_pool.go | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/core/tx_pool.go b/core/tx_pool.go index 91ceea4bbe..b4e3191f7a 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -23,6 +23,7 @@ import ( "math/big" "sort" "sync" + "sync/atomic" "time" "github.com/tomochain/tomochain/common" @@ -205,7 +206,7 @@ func (config *TxPoolConfig) sanitize() TxPoolConfig { // two states over time as they are received and processed. type TxPool struct { config TxPoolConfig - chainconfig *params.ChainConfig + chainConfig *params.ChainConfig chain blockChain gasPrice *big.Int txFeed event.Feed @@ -215,9 +216,9 @@ type TxPool struct { signer types.Signer mu sync.RWMutex - currentState *state.StateDB // Current state in the blockchain head - pendingState *state.ManagedState // Pending state tracking virtual nonces - currentMaxGas uint64 // Current gas limit for transaction caps + currentHead atomic.Pointer[types.Header] // Current head of the blockchain + currentState *state.StateDB // Current state in the blockchain head + pendingState *state.ManagedState // Pending state tracking virtual nonces locals *accountSet // Set of local transaction to exempt from eviction rules journal *txJournal // Journal of local transaction to back up to disk @@ -244,7 +245,7 @@ func NewTxPool(config TxPoolConfig, chainconfig *params.ChainConfig, chain block // Create the transaction pool with its initial settings pool := &TxPool{ config: config, - chainconfig: chainconfig, + chainConfig: chainconfig, chain: chain, signer: types.LatestSigner(chainconfig), pending: make(map[common.Address]*txList), @@ -308,7 +309,7 @@ func (pool *TxPool) loop() { case ev := <-pool.chainHeadCh: if ev.Block != nil { pool.mu.Lock() - if pool.chainconfig.IsHomestead(ev.Block.Number()) { + if pool.chainConfig.IsHomestead(ev.Block.Number()) { pool.homestead = true } pool.reset(head.Header(), ev.Block.Header()) @@ -430,10 +431,10 @@ func (pool *TxPool) reset(oldHead, newHead *types.Header) { log.Error("Failed to reset txpool state", "err", err) return } + pool.currentHead.Store(newHead) pool.currentState = statedb pool.trc21FeeCapacity = state.GetTRC21FeeCapacityFromStateWithCache(newHead.Root, statedb) pool.pendingState = state.ManageState(statedb) - pool.currentMaxGas = newHead.GasLimit // Inject any transactions discarded due to reorgs log.Debug("Reinjecting stale transactions", "count", len(reinject)) @@ -586,6 +587,7 @@ func (pool *TxPool) GetSender(tx *types.Transaction) (common.Address, error) { // validateTx checks whether a transaction is valid according to the consensus // rules and adheres to some heuristic limits of the local node (price and size). func (pool *TxPool) validateTx(tx *types.Transaction, local bool) error { + head := pool.currentHead.Load() // check if sender is in black list if tx.From() != nil && common.Blacklist[*tx.From()] { return fmt.Errorf("Reject transaction with sender in black-list: %v", tx.From().Hex()) @@ -599,15 +601,29 @@ func (pool *TxPool) validateTx(tx *types.Transaction, local bool) error { if tx.Size() > 32*1024 { return ErrOversizedData } + if !pool.chainConfig.IsLondon(head.Number) && tx.Type() != types.LegacyTxType { + return fmt.Errorf("%w: type %d rejected, pool not yet in London", types.ErrTxTypeNotSupported, tx.Type()) + } // Transactions can't be negative. This may never happen using RLP decoded // transactions but may occur if you create a transaction using the RPC. if tx.Value().Sign() < 0 { return ErrNegativeValue } // Ensure the transaction doesn't exceed the current block limit gas. - if pool.currentMaxGas < tx.Gas() { + if head.GasLimit < tx.Gas() { return ErrGasLimit } + // Sanity check for extremely large numbers (supported by RLP or RPC) + if tx.GasFeeCap().BitLen() > 256 { + return ErrFeeCapVeryHigh + } + if tx.GasTipCap().BitLen() > 256 { + return ErrTipVeryHigh + } + // Ensure gasFeeCap is greater than or equal to gasTipCap + if tx.GasFeeCapIntCmp(tx.GasTipCap()) < 0 { + return ErrTipAboveFeeCap + } // Make sure the transaction is signed properly from, err := types.Sender(pool.signer, tx) if err != nil { @@ -1052,6 +1068,7 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) { } } // Iterate over all accounts and promote any executable transactions + gasLimit := pool.currentHead.Load().GasLimit for _, addr := range accounts { list := pool.queue[addr] if list == nil { @@ -1065,7 +1082,7 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) { pool.priced.Removed() } // Drop all transactions that are too costly (low balance or out of gas) - drops, _ := list.Filter(pool.currentState.GetBalance(addr), pool.currentMaxGas, pool.trc21FeeCapacity) + drops, _ := list.Filter(pool.currentState.GetBalance(addr), gasLimit, pool.trc21FeeCapacity) for _, tx := range drops { hash := tx.Hash() log.Trace("Removed unpayable queued transaction", "hash", hash) @@ -1212,6 +1229,7 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) { // are moved back into the future queue. func (pool *TxPool) demoteUnexecutables() { // Iterate over all accounts and demote any non-executable transactions + gasLimit := pool.currentHead.Load().GasLimit for addr, list := range pool.pending { nonce := pool.currentState.GetNonce(addr) @@ -1223,7 +1241,7 @@ func (pool *TxPool) demoteUnexecutables() { pool.priced.Removed() } // Drop all transactions that are too costly (low balance or out of gas), and queue any invalids back for later - drops, invalids := list.Filter(pool.currentState.GetBalance(addr), pool.currentMaxGas, pool.trc21FeeCapacity) + drops, invalids := list.Filter(pool.currentState.GetBalance(addr), gasLimit, pool.trc21FeeCapacity) for _, tx := range drops { hash := tx.Hash() log.Trace("Removed unpayable pending transaction", "hash", hash) From eee58511ac8ffa860d94656644ebfdb07e879b4e Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Fri, 30 Jun 2023 11:09:04 +0700 Subject: [PATCH 37/82] Calculate intrinsic gas of access list tx --- core/bench_test.go | 4 ++-- core/state_transition.go | 8 ++++++-- core/tx_pool.go | 2 +- light/txpool.go | 11 +++++++---- params/protocol_params.go | 10 ++++++---- 5 files changed, 22 insertions(+), 13 deletions(-) diff --git a/core/bench_test.go b/core/bench_test.go index 137b57f031..a52aa7f2c2 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -18,7 +18,6 @@ package core import ( "crypto/ecdsa" - "github.com/tomochain/tomochain/core/rawdb" "io/ioutil" "math/big" "os" @@ -27,6 +26,7 @@ import ( "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/common/math" "github.com/tomochain/tomochain/consensus/ethash" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/core/vm" "github.com/tomochain/tomochain/crypto" @@ -85,7 +85,7 @@ func genValueTx(nbytes int) func(int, *BlockGen) { return func(i int, gen *BlockGen) { toaddr := common.Address{} data := make([]byte, nbytes) - gas, _ := IntrinsicGas(data, false, false) + gas, _ := IntrinsicGas(data, nil, false, false) tx, _ := types.SignTx(types.NewTransaction(gen.TxNonce(benchRootAddr), toaddr, big.NewInt(1), gas, nil, data), types.HomesteadSigner{}, benchRootKey) gen.AddTx(tx) } diff --git a/core/state_transition.go b/core/state_transition.go index 8eef0a1b27..79463fe93c 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -87,7 +87,7 @@ type Message struct { } // IntrinsicGas computes the 'intrinsic gas' for a message with the given data. -func IntrinsicGas(data []byte, contractCreation, homestead bool) (uint64, error) { +func IntrinsicGas(data []byte, accessList types.AccessList, contractCreation, homestead bool) (uint64, error) { // Set the starting gas for the raw transaction var gas uint64 if contractCreation && homestead { @@ -116,6 +116,10 @@ func IntrinsicGas(data []byte, contractCreation, homestead bool) (uint64, error) } gas += z * params.TxDataZeroGas } + if accessList != nil { + gas += uint64(len(accessList)) * params.TxAccessListAddressGas + gas += uint64(accessList.StorageKeys()) * params.TxAccessListStorageKeyGas + } return gas, nil } @@ -296,7 +300,7 @@ func (st *StateTransition) TransitionDb(owner common.Address) (ret []byte, usedG contractCreation := msg.To == nil // Pay intrinsic gas - gas, err := IntrinsicGas(st.data, contractCreation, homestead) + gas, err := IntrinsicGas(st.data, msg.AccessList, contractCreation, homestead) if err != nil { return nil, 0, false, err } diff --git a/core/tx_pool.go b/core/tx_pool.go index b4e3191f7a..83412e4fcc 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -665,7 +665,7 @@ func (pool *TxPool) validateTx(tx *types.Transaction, local bool) error { } if tx.To() == nil || (tx.To() != nil && !tx.IsSpecialTransaction()) { - intrGas, err := IntrinsicGas(tx.Data(), tx.To() == nil, pool.homestead) + intrGas, err := IntrinsicGas(tx.Data(), tx.AccessList(), tx.To() == nil, pool.homestead) if err != nil { return err } diff --git a/light/txpool.go b/light/txpool.go index 7af86dbd6b..0859b88dc0 100644 --- a/light/txpool.go +++ b/light/txpool.go @@ -74,10 +74,13 @@ type TxPool struct { // // Send instructs backend to forward new transactions // NewHead notifies backend about a new head after processed by the tx pool, -// including mined and rolled back transactions since the last event +// +// including mined and rolled back transactions since the last event +// // Discard notifies backend about transactions that should be discarded either -// because they have been replaced by a re-send or because they have been mined -// long ago and no rollback is expected +// +// because they have been replaced by a re-send or because they have been mined +// long ago and no rollback is expected type TxRelayBackend interface { Send(txs types.Transactions) NewHead(head common.Hash, mined []common.Hash, rollback []common.Hash) @@ -400,7 +403,7 @@ func (pool *TxPool) validateTx(ctx context.Context, tx *types.Transaction) error } // Should supply enough intrinsic gas - gas, err := core.IntrinsicGas(tx.Data(), tx.To() == nil, pool.homestead) + gas, err := core.IntrinsicGas(tx.Data(), tx.AccessList(), tx.To() == nil, pool.homestead) if err != nil { return err } diff --git a/params/protocol_params.go b/params/protocol_params.go index 86c8d88af7..4bb9270a3a 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -107,10 +107,12 @@ const ( SstoreCleanRefundEIP2200 uint64 = 4200 // Once per SSTORE operation for resetting to the original non-zero value SstoreClearRefundEIP2200 uint64 = 15000 // Once per SSTORE operation for clearing an originally existing storage slot - Create2Gas uint64 = 32000 // Once per CREATE2 operation - SelfdestructRefundGas uint64 = 24000 // Refunded following a selfdestruct operation. - TxDataNonZeroGasFrontier uint64 = 68 // Per byte of data attached to a transaction that is not equal to zero. NOTE: Not payable on data of calls between transactions. - TxDataNonZeroGasEIP2028 uint64 = 16 // Per byte of non zero data attached to a transaction after EIP 2028 (part in Istanbul) + Create2Gas uint64 = 32000 // Once per CREATE2 operation + SelfdestructRefundGas uint64 = 24000 // Refunded following a selfdestruct operation. + TxDataNonZeroGasFrontier uint64 = 68 // Per byte of data attached to a transaction that is not equal to zero. NOTE: Not payable on data of calls between transactions. + TxDataNonZeroGasEIP2028 uint64 = 16 // Per byte of non zero data attached to a transaction after EIP 2028 (part in Istanbul) + TxAccessListAddressGas uint64 = 2400 // Per address specified in EIP 2930 access list + TxAccessListStorageKeyGas uint64 = 1900 // Per storage key specified in EIP 2930 access list // These have been changed during the course of the chain CallGasFrontier uint64 = 40 // Once per CALL operation & message call transaction. From 4ccb1ca363307b91a7c600b9180727ef42a339f3 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Fri, 30 Jun 2023 14:41:06 +0700 Subject: [PATCH 38/82] Fix hashing transactions --- core/state_processor.go | 1 - core/tx_pool.go | 1 - core/types/transaction.go | 18 +++++++++--------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/core/state_processor.go b/core/state_processor.go index 60cffbd41d..1bb5044369 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -517,7 +517,6 @@ func InitSignerInTransactions(config *params.ChainConfig, header *types.Header, go func(from int, to int) { for j := from; j < to; j++ { types.CacheSigner(signer, txs[j]) - txs[j].CacheHash() } wg.Done() }(from, to) diff --git a/core/tx_pool.go b/core/tx_pool.go index 83412e4fcc..9f946ba90a 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -929,7 +929,6 @@ func (pool *TxPool) AddRemotes(txs []*types.Transaction) []error { // addTx enqueues a single transaction into the pool if it is valid. func (pool *TxPool) addTx(tx *types.Transaction, local bool) error { - tx.CacheHash() types.CacheSigner(pool.signer, tx) pool.mu.Lock() defer pool.mu.Unlock() diff --git a/core/types/transaction.go b/core/types/transaction.go index 4ed3bf614d..06ff61a6f9 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -339,20 +339,20 @@ func (tx *Transaction) From() *common.Address { } } -// Hash hashes the RLP encoding of tx. -// It uniquely identifies the transaction. +// Hash returns the transaction hash. func (tx *Transaction) Hash() common.Hash { if hash := tx.hash.Load(); hash != nil { return hash.(common.Hash) } - v := rlpHash(tx) - tx.hash.Store(v) - return v -} -func (tx *Transaction) CacheHash() { - v := rlpHash(tx) - tx.hash.Store(v) + var h common.Hash + if tx.Type() == LegacyTxType { + h = rlpHash(tx.inner) + } else { + h = prefixedRlpHash(tx.Type(), tx.inner) + } + tx.hash.Store(h) + return h } // Size returns the true RLP encoded storage size of the transaction, either by From 0d54f0d992e337f86a9d4fed86ca7bde60dbf2a7 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Fri, 30 Jun 2023 15:24:42 +0700 Subject: [PATCH 39/82] Fix transactions' size --- core/types/transaction.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/core/types/transaction.go b/core/types/transaction.go index 06ff61a6f9..98f22e18be 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -355,16 +355,21 @@ func (tx *Transaction) Hash() common.Hash { return h } -// Size returns the true RLP encoded storage size of the transaction, either by -// encoding and returning it, or returning a previsouly cached value. +// Size returns the true encoded storage size of the transaction, either by encoding +// and returning it, or returning a previously cached value. func (tx *Transaction) Size() common.StorageSize { if size := tx.size.Load(); size != nil { return size.(common.StorageSize) } c := writeCounter(0) rlp.Encode(&c, &tx.inner) - tx.size.Store(common.StorageSize(c)) - return common.StorageSize(c) + + size := common.StorageSize(c) + if tx.Type() != LegacyTxType { + size += 1 // type byte + } + tx.size.Store(size) + return size } // WithSignature returns a new transaction with the given signature. From b122c91b05b3dc681aa5ce5a396d5f16325916b5 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Mon, 3 Jul 2023 15:06:48 +0700 Subject: [PATCH 40/82] Specify EIP-1559 gas field when call contract in simulatedBackend --- accounts/abi/bind/auth.go | 42 ++++++++++++- accounts/abi/bind/backend.go | 3 + accounts/abi/bind/backends/simulated.go | 80 +++++++++++++++++++++++-- accounts/abi/bind/base.go | 76 ++++++++++------------- 4 files changed, 149 insertions(+), 52 deletions(-) diff --git a/accounts/abi/bind/auth.go b/accounts/abi/bind/auth.go index 6c68a4e7a8..12b1607a93 100644 --- a/accounts/abi/bind/auth.go +++ b/accounts/abi/bind/auth.go @@ -17,17 +17,26 @@ package bind import ( + "context" "crypto/ecdsa" "errors" "io" "io/ioutil" + "math/big" "github.com/tomochain/tomochain/accounts/keystore" "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/crypto" + "github.com/tomochain/tomochain/log" ) +// ErrNoChainID is returned whenever the user failed to specify a chain id. +var ErrNoChainID = errors.New("no chain id specified") + +// ErrNotAuthorized is returned when an account is not properly unlocked. +var ErrNotAuthorized = errors.New("not authorized to sign this account") + // NewTransactor is a utility method to easily create a transaction signer from // an encrypted json key stream and the associated passphrase. func NewTransactor(keyin io.Reader, passphrase string) (*TransactOpts, error) { @@ -44,13 +53,17 @@ func NewTransactor(keyin io.Reader, passphrase string) (*TransactOpts, error) { // NewKeyedTransactor is a utility method to easily create a transaction signer // from a single private key. +// +// Deprecated: Use NewKeyedTransactorWithChainID instead. func NewKeyedTransactor(key *ecdsa.PrivateKey) *TransactOpts { + log.Warn("WARNING: NewKeyedTransactor has been deprecated in favour of NewKeyedTransactorWithChainID") keyAddr := crypto.PubkeyToAddress(key.PublicKey) + signer := types.HomesteadSigner{} return &TransactOpts{ From: keyAddr, - Signer: func(signer types.Signer, address common.Address, tx *types.Transaction) (*types.Transaction, error) { + Signer: func(address common.Address, tx *types.Transaction) (*types.Transaction, error) { if address != keyAddr { - return nil, errors.New("not authorized to sign this account") + return nil, ErrNotAuthorized } signature, err := crypto.Sign(signer.Hash(tx).Bytes(), key) if err != nil { @@ -58,5 +71,30 @@ func NewKeyedTransactor(key *ecdsa.PrivateKey) *TransactOpts { } return tx.WithSignature(signer, signature) }, + Context: context.Background(), } } + +// NewKeyedTransactorWithChainID is a utility method to easily create a transaction signer +// from a single private key. +func NewKeyedTransactorWithChainID(key *ecdsa.PrivateKey, chainID *big.Int) (*TransactOpts, error) { + keyAddr := crypto.PubkeyToAddress(key.PublicKey) + if chainID == nil { + return nil, ErrNoChainID + } + signer := types.LatestSignerForChainID(chainID) + return &TransactOpts{ + From: keyAddr, + Signer: func(address common.Address, tx *types.Transaction) (*types.Transaction, error) { + if address != keyAddr { + return nil, ErrNotAuthorized + } + signature, err := crypto.Sign(signer.Hash(tx).Bytes(), key) + if err != nil { + return nil, err + } + return tx.WithSignature(signer, signature) + }, + Context: context.Background(), + }, nil +} diff --git a/accounts/abi/bind/backend.go b/accounts/abi/bind/backend.go index ee246d007e..aa6cbfce82 100644 --- a/accounts/abi/bind/backend.go +++ b/accounts/abi/bind/backend.go @@ -67,6 +67,9 @@ type PendingContractCaller interface { // used when the user does not provide some needed values, but rather leaves it up // to the transactor to decide. type ContractTransactor interface { + // HeaderByNumber returns a block header from the current canonical chain. If + // number is nil, the latest known header is returned. + HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) // PendingCodeAt returns the code of the given account in the pending state. PendingCodeAt(ctx context.Context, account common.Address) ([]byte, error) // PendingNonceAt retrieves the current pending nonce associated with an account. diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index cbf09cdcaa..6da52e0752 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -39,6 +39,7 @@ import ( "github.com/tomochain/tomochain/eth/filters" "github.com/tomochain/tomochain/ethdb" "github.com/tomochain/tomochain/event" + "github.com/tomochain/tomochain/log" "github.com/tomochain/tomochain/params" "github.com/tomochain/tomochain/rpc" ) @@ -178,6 +179,19 @@ func (b *SimulatedBackend) TransactionReceipt(ctx context.Context, txHash common return receipt, nil } +// HeaderByNumber returns a block header from the current canonical chain. If number is +// nil, the latest known header is returned. +func (b *SimulatedBackend) HeaderByNumber(ctx context.Context, block *big.Int) (*types.Header, error) { + b.mu.Lock() + defer b.mu.Unlock() + + if block == nil || block.Cmp(b.pendingBlock.Number()) == 0 { + return b.blockchain.CurrentHeader(), nil + } + + return b.blockchain.GetHeaderByNumber(uint64(block.Int64())), nil +} + // PendingCodeAt returns the code associated with an account in the pending state. func (b *SimulatedBackend) PendingCodeAt(ctx context.Context, contract common.Address) ([]byte, error) { b.mu.Lock() @@ -294,6 +308,38 @@ func (b *SimulatedBackend) EstimateGas(ctx context.Context, call tomochain.CallM } else { hi = b.pendingBlock.GasLimit() } + // Normalize the max fee per gas the call is willing to spend. + var feeCap *big.Int + if call.GasPrice != nil && (call.GasFeeCap != nil || call.GasTipCap != nil) { + return 0, errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") + } else if call.GasPrice != nil { + feeCap = call.GasPrice + } else if call.GasFeeCap != nil { + feeCap = call.GasFeeCap + } else { + feeCap = common.Big0 + } + // Recap the highest gas allowance with account's balance. + if feeCap.BitLen() != 0 { + balance := b.pendingState.GetBalance(call.From) // from can't be nil + available := new(big.Int).Set(balance) + if call.Value != nil { + if call.Value.Cmp(available) >= 0 { + return 0, core.ErrInsufficientFundsForTransfer + } + available.Sub(available, call.Value) + } + allowance := new(big.Int).Div(available, feeCap) + if allowance.IsUint64() && hi > allowance.Uint64() { + transfer := call.Value + if transfer == nil { + transfer = new(big.Int) + } + log.Warn("Gas estimation capped by limited funds", "original", hi, "balance", balance, + "sent", transfer, "feecap", feeCap, "fundable", allowance) + hi = allowance.Uint64() + } + } cap = hi // Create a helper to check if a gas allowance results in an executable transaction @@ -302,7 +348,6 @@ func (b *SimulatedBackend) EstimateGas(ctx context.Context, call tomochain.CallM snapshot := b.pendingState.Snapshot() _, _, failed, err := b.callContract(ctx, call, b.pendingBlock, b.pendingState) - fmt.Println("EstimateGas", err, failed) b.pendingState.RevertToSnapshot(snapshot) if err != nil || failed { @@ -331,9 +376,36 @@ func (b *SimulatedBackend) EstimateGas(ctx context.Context, call tomochain.CallM // callContract implements common code between normal and pending contract calls. // state is modified during execution, make sure to copy it if necessary. func (b *SimulatedBackend) callContract(ctx context.Context, call tomochain.CallMsg, block *types.Block, statedb *state.StateDB) ([]byte, uint64, bool, error) { - // Ensure message is initialized properly. - if call.GasPrice == nil { - call.GasPrice = big.NewInt(1) + // Gas prices post 1559 need to be initialized + if call.GasPrice != nil && (call.GasFeeCap != nil || call.GasTipCap != nil) { + return nil, 0, false, errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") + } + head := b.blockchain.CurrentHeader() + if !b.blockchain.Config().IsLondon(head.Number) { + // If there's no basefee, then it must be a non-1559 execution + if call.GasPrice == nil { + call.GasPrice = new(big.Int) + } + call.GasFeeCap, call.GasTipCap = call.GasPrice, call.GasPrice + } else { + // A basefee is provided, necessitating 1559-type execution + if call.GasPrice != nil { + // User specified the legacy gas field, convert to 1559 gas typing + call.GasFeeCap, call.GasTipCap = call.GasPrice, call.GasPrice + } else { + // User specified 1559 gas fields (or none), use those + if call.GasFeeCap == nil { + call.GasFeeCap = new(big.Int) + } + if call.GasTipCap == nil { + call.GasTipCap = new(big.Int) + } + // Backfill the legacy gasPrice for EVM execution, unless we're all zeroes + call.GasPrice = new(big.Int) + if call.GasFeeCap.BitLen() > 0 || call.GasTipCap.BitLen() > 0 { + call.GasPrice = math.BigMin(new(big.Int).Add(call.GasTipCap, head.BaseFee), call.GasFeeCap) + } + } } if call.Gas == 0 { call.Gas = 50000000 diff --git a/accounts/abi/bind/base.go b/accounts/abi/bind/base.go index 596951553a..8d5b31cd59 100644 --- a/accounts/abi/bind/base.go +++ b/accounts/abi/bind/base.go @@ -34,7 +34,7 @@ const basefeeWiggleMultiplier = 2 // SignerFn is a signer function callback when a contract requires a method to // sign the transaction before submission. -type SignerFn func(types.Signer, common.Address, *types.Transaction) (*types.Transaction, error) +type SignerFn func(common.Address, *types.Transaction) (*types.Transaction, error) // CallOpts is the collection of options to fine tune a contract call request. type CallOpts struct { @@ -58,6 +58,8 @@ type TransactOpts struct { GasLimit uint64 // Gas limit to set for the transaction execution (0 = estimate) Context context.Context // Network context to support cancellation and timeouts (nil = no timeout) + + NoSend bool // Do all transact steps but do not send the transaction } // FilterOpts is the collection of options to fine tune filtering for events @@ -81,9 +83,9 @@ type WatchOpts struct { // higher level contract bindings to operate. type BoundContract struct { address common.Address // Deployment address of the contract on the Ethereum blockchain + transactor ContractTransactor // Write interface to interact with the blockchain abi abi.ABI // Reflect based ABI to access the correct Ethereum methods caller ContractCaller // Read interface to interact with the blockchain - transactor ContractTransactor // Write interface to interact with the blockchain filterer ContractFilterer // Event filtering to interact with the blockchain } @@ -311,61 +313,43 @@ func (c *BoundContract) getNonce(opts *TransactOpts) (uint64, error) { // transact executes an actual transaction invocation, first deriving any missing // authorization fields, and then scheduling the transaction for execution. func (c *BoundContract) transact(opts *TransactOpts, contract *common.Address, input []byte) (*types.Transaction, error) { - var err error - - // Ensure a valid value field and resolve the account nonce - value := opts.Value - if value == nil { - value = new(big.Int) + if opts.GasPrice != nil && (opts.GasFeeCap != nil || opts.GasTipCap != nil) { + return nil, errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") } - var nonce uint64 - if opts.Nonce == nil { - nonce, err = c.transactor.PendingNonceAt(ensureContext(opts.Context), opts.From) - if err != nil { - return nil, fmt.Errorf("failed to retrieve account nonce: %v", err) - } + // Create the transaction + var ( + rawTx *types.Transaction + err error + ) + if opts.GasPrice != nil { + rawTx, err = c.createLegacyTx(opts, contract, input) + } else if opts.GasFeeCap != nil && opts.GasTipCap != nil { + rawTx, err = c.createDynamicTx(opts, contract, input, nil) } else { - nonce = opts.Nonce.Uint64() - } - // Figure out the gas allowance and gas price values - gasPrice := opts.GasPrice - if gasPrice == nil { - gasPrice, err = c.transactor.SuggestGasPrice(ensureContext(opts.Context)) - if err != nil { - return nil, fmt.Errorf("failed to suggest gas price: %v", err) + // Only query for basefee if gasPrice not specified + if head, errHead := c.transactor.HeaderByNumber(ensureContext(opts.Context), nil); errHead != nil { + return nil, errHead + } else if head.BaseFee != nil { + rawTx, err = c.createDynamicTx(opts, contract, input, head) + } else { + // Chain is not London ready -> use legacy transaction + rawTx, err = c.createLegacyTx(opts, contract, input) } } - gasLimit := opts.GasLimit - if gasLimit == 0 { - // Gas estimation cannot succeed without code for method invocations - if contract != nil { - if code, err := c.transactor.PendingCodeAt(ensureContext(opts.Context), c.address); err != nil { - return nil, err - } else if len(code) == 0 { - return nil, ErrNoCode - } - } - // If the contract surely has code (or code is not needed), estimate the transaction - msg := tomochain.CallMsg{From: opts.From, To: contract, Value: value, Data: input} - gasLimit, err = c.transactor.EstimateGas(ensureContext(opts.Context), msg) - if err != nil { - return nil, fmt.Errorf("failed to estimate gas needed: %v", err) - } - } - // Create the transaction, sign it and schedule it for execution - var rawTx *types.Transaction - if contract == nil { - rawTx = types.NewContractCreation(nonce, value, gasLimit, gasPrice, input) - } else { - rawTx = types.NewTransaction(nonce, c.address, value, gasLimit, gasPrice, input) + if err != nil { + return nil, err } + // Sign the transaction and schedule it for execution if opts.Signer == nil { return nil, errors.New("no signer to authorize the transaction with") } - signedTx, err := opts.Signer(types.HomesteadSigner{}, opts.From, rawTx) + signedTx, err := opts.Signer(opts.From, rawTx) if err != nil { return nil, err } + if opts.NoSend { + return signedTx, nil + } if err := c.transactor.SendTransaction(ensureContext(opts.Context), signedTx); err != nil { return nil, err } From 25d7be8c57088c8fba2dcd413d3db55030f93808 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Mon, 3 Jul 2023 15:13:19 +0700 Subject: [PATCH 41/82] Fix mobile --- mobile/bind.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mobile/bind.go b/mobile/bind.go index 606a26ae79..0c05447959 100644 --- a/mobile/bind.go +++ b/mobile/bind.go @@ -39,7 +39,7 @@ type signer struct { } func (s *signer) Sign(addr *Address, unsignedTx *Transaction) (signedTx *Transaction, _ error) { - sig, err := s.sign(types.HomesteadSigner{}, addr.address, unsignedTx.tx) + sig, err := s.sign(addr.address, unsignedTx.tx) if err != nil { return nil, err } @@ -89,7 +89,7 @@ func (opts *TransactOpts) GetGasLimit() int64 { return int64(opts.opts.GasLimi func (opts *TransactOpts) SetFrom(from *Address) { opts.opts.From = from.address } func (opts *TransactOpts) SetNonce(nonce int64) { opts.opts.Nonce = big.NewInt(nonce) } func (opts *TransactOpts) SetSigner(s Signer) { - opts.opts.Signer = func(signer types.Signer, addr common.Address, tx *types.Transaction) (*types.Transaction, error) { + opts.opts.Signer = func(addr common.Address, tx *types.Transaction) (*types.Transaction, error) { sig, err := s.Sign(&Address{addr}, &Transaction{tx}) if err != nil { return nil, err From 1218d2c1a988a63015493bb13d23a9fad6f00283 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Mon, 3 Jul 2023 17:16:29 +0700 Subject: [PATCH 42/82] Implement BlockNumberOrHash and some api backend methods --- eth/api_backend.go | 42 +++++++++++++ internal/ethapi/api.go | 94 +++++++++++++++++++++++++++- internal/ethapi/backend.go | 3 + les/api_backend.go | 41 +++++++++++++ rpc/types.go | 122 +++++++++++++++++++++++++++++++++++-- 5 files changed, 297 insertions(+), 5 deletions(-) diff --git a/eth/api_backend.go b/eth/api_backend.go index cd1d589545..8b6a4826f5 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -85,6 +85,27 @@ func (b *EthApiBackend) HeaderByNumber(ctx context.Context, blockNr rpc.BlockNum return b.eth.blockchain.GetHeaderByNumber(uint64(blockNr)), nil } +func (b *EthApiBackend) HeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error) { + if blockNr, ok := blockNrOrHash.Number(); ok { + return b.HeaderByNumber(ctx, blockNr) + } + if hash, ok := blockNrOrHash.Hash(); ok { + header := b.eth.blockchain.GetHeaderByHash(hash) + if header == nil { + return nil, errors.New("header for hash not found") + } + if blockNrOrHash.RequireCanonical && core.GetCanonicalHash(b.ChainDb(), header.Number.Uint64()) != hash { + return nil, errors.New("hash is not currently canonical") + } + return header, nil + } + return nil, errors.New("invalid arguments; neither block nor hash specified") +} + +func (b *EthApiBackend) HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) { + return b.eth.blockchain.GetHeaderByHash(hash), nil +} + func (b *EthApiBackend) BlockByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Block, error) { // Pending block is only known by the miner if blockNr == rpc.PendingBlockNumber { @@ -113,6 +134,27 @@ func (b *EthApiBackend) StateAndHeaderByNumber(ctx context.Context, blockNr rpc. return stateDb, header, err } +func (b *EthApiBackend) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) { + if blockNr, ok := blockNrOrHash.Number(); ok { + return b.StateAndHeaderByNumber(ctx, blockNr) + } + if hash, ok := blockNrOrHash.Hash(); ok { + header, err := b.HeaderByHash(ctx, hash) + if err != nil { + return nil, nil, err + } + if header == nil { + return nil, nil, errors.New("header for hash not found") + } + if blockNrOrHash.RequireCanonical && core.GetCanonicalHash(b.ChainDb(), header.Number.Uint64()) != hash { + return nil, nil, errors.New("hash is not currently canonical") + } + stateDb, err := b.eth.BlockChain().StateAt(header.Root) + return stateDb, header, err + } + return nil, nil, errors.New("invalid arguments; neither block nor hash specified") +} + func (b *EthApiBackend) GetBlock(ctx context.Context, blockHash common.Hash) (*types.Block, error) { return b.eth.blockchain.GetBlockByHash(blockHash), nil } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index ef4439241d..8af445fe11 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1053,7 +1053,7 @@ func (s *PublicBlockChainAPI) doCall(ctx context.Context, args TransactionArgs, if gasPrice == nil { gasPrice = args.MaxFeePerGas.ToInt() } - if gasPrice.Sign() == 0 { + if gasPrice == nil || gasPrice.Sign() == 0 { gasPrice = new(big.Int).SetUint64(defaultGasPrice) } balanceTokenFee := big.NewInt(0).SetUint64(uint64(*args.Gas)) @@ -1607,6 +1607,97 @@ func newRPCRawTransactionFromBlockIndex(b *types.Block, index uint64) hexutil.By return blob } +// accessListResult returns an optional accesslist +// It's the result of the `debug_createAccessList` RPC call. +// It contains an error if the transaction itself failed. +type accessListResult struct { + Accesslist *types.AccessList `json:"accessList"` + Error string `json:"error,omitempty"` + GasUsed hexutil.Uint64 `json:"gasUsed"` +} + +// CreateAccessList creates an EIP-2930 type AccessList for the given transaction. +// Reexec and BlockNrOrHash can be specified to create the accessList on top of a certain state. +func (s *PublicBlockChainAPI) CreateAccessList(ctx context.Context, args TransactionArgs, blockNrOrHash *rpc.BlockNumberOrHash) (*accessListResult, error) { + bNrOrHash := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) + if blockNrOrHash != nil { + bNrOrHash = *blockNrOrHash + } + acl, gasUsed, vmerr, err := AccessList(ctx, s.b, bNrOrHash, args) + if err != nil { + return nil, err + } + result := &accessListResult{Accesslist: &acl, GasUsed: hexutil.Uint64(gasUsed)} + if vmerr != nil { + result.Error = vmerr.Error() + } + return result, nil +} + +// AccessList creates an access list for the given transaction. +// If the accesslist creation fails an error is returned. +// If the transaction itself fails, an vmErr is returned. +func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrHash, args TransactionArgs) (acl types.AccessList, gasUsed uint64, vmErr error, err error) { + return nil, 0, nil, fmt.Errorf("failed to apply transaction: %v err: %v", args.toTransaction().Hash(), err) + //// Retrieve the execution context + //db, header, err := b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) + //if db == nil || err != nil { + // return nil, 0, nil, err + //} + //// If the gas amount is not set, default to RPC gas cap. + //if args.Gas == nil { + // tmp := hexutil.Uint64(params.MinGasLimit) + // args.Gas = &tmp + //} + // + //// Ensure any missing fields are filled, extract the recipient and input data + //if err := args.setDefaults(ctx, b); err != nil { + // return nil, 0, nil, err + //} + //var to common.Address + //if args.To != nil { + // to = *args.To + //} else { + // to = crypto.CreateAddress(args.from(), uint64(*args.Nonce)) + //} + //isPostMerge := header.Difficulty.Cmp(common.Big0) == 0 + //// Retrieve the precompiles since they don't need to be added to the access list + //precompiles := vm.ActivePrecompiles(b.ChainConfig().Rules(header.Number, isPostMerge, header.Time)) + // + //// Create an initial tracer + //prevTracer := logger.NewAccessListTracer(nil, args.from(), to, precompiles) + //if args.AccessList != nil { + // prevTracer = logger.NewAccessListTracer(*args.AccessList, args.from(), to, precompiles) + //} + //for { + // // Retrieve the current access list to expand + // accessList := prevTracer.AccessList() + // log.Trace("Creating access list", "input", accessList) + // + // // Copy the original db so we don't modify it + // statedb := db.Copy() + // // Set the accesslist to the last al + // args.AccessList = &accessList + // msg, err := args.ToMessage(header.BaseFee) + // if err != nil { + // return nil, 0, nil, err + // } + // + // // Apply the transaction with the access list tracer + // tracer := logger.NewAccessListTracer(accessList, args.from(), to, precompiles) + // config := vm.Config{Tracer: tracer, NoBaseFee: true} + // vmenv, _ := b.GetEVM(ctx, msg, statedb, header, &config, nil) + // res, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.GasLimit)) + // if err != nil { + // return nil, 0, nil, fmt.Errorf("failed to apply transaction: %v err: %v", args.toTransaction().Hash(), err) + // } + // if tracer.Equal(prevTracer) { + // return accessList, res.UsedGas, res.Err, nil + // } + // prevTracer = tracer + //} +} + // PublicTransactionPoolAPI exposes methods for the RPC interface type PublicTransactionPoolAPI struct { b Backend @@ -2159,6 +2250,7 @@ func (s *PublicTransactionPoolAPI) SendRawTransaction(ctx context.Context, encod if err := tx.UnmarshalBinary(encodedTx); err != nil { return common.Hash{}, err } + fmt.Printf("@@@@@@@@@@@@@@@@@@@@@@@@@@@@ SendRawTransaction: %+v\n", tx.Hash().Hex()) return submitTransaction(ctx, s.b, tx) } diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index a043524029..a3461db669 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -55,9 +55,12 @@ type Backend interface { // BlockChain API SetHead(number uint64) HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) + HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) + HeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error) BlockByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Block, error) CurrentHeader() *types.Header StateAndHeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*state.StateDB, *types.Header, error) + StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) GetBlock(ctx context.Context, blockHash common.Hash) (*types.Block, error) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) GetTd(blockHash common.Hash) *big.Int diff --git a/les/api_backend.go b/les/api_backend.go index 66f759472c..f433b83ffc 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -72,6 +72,30 @@ func (b *LesApiBackend) HeaderByNumber(ctx context.Context, blockNr rpc.BlockNum return b.eth.blockchain.GetHeaderByNumberOdr(ctx, uint64(blockNr)) } +func (b *LesApiBackend) HeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error) { + if blockNr, ok := blockNrOrHash.Number(); ok { + return b.HeaderByNumber(ctx, blockNr) + } + if hash, ok := blockNrOrHash.Hash(); ok { + header, err := b.HeaderByHash(ctx, hash) + if err != nil { + return nil, err + } + if header == nil { + return nil, errors.New("header for hash not found") + } + if blockNrOrHash.RequireCanonical && core.GetCanonicalHash(b.ChainDb(), header.Number.Uint64()) != hash { + return nil, errors.New("hash is not currently canonical") + } + return header, nil + } + return nil, errors.New("invalid arguments; neither block nor hash specified") +} + +func (b *LesApiBackend) HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) { + return b.eth.blockchain.GetHeaderByHash(hash), nil +} + func (b *LesApiBackend) BlockByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Block, error) { header, err := b.HeaderByNumber(ctx, blockNr) if header == nil || err != nil { @@ -88,6 +112,23 @@ func (b *LesApiBackend) StateAndHeaderByNumber(ctx context.Context, blockNr rpc. return light.NewState(ctx, header, b.eth.odr), header, nil } +func (b *LesApiBackend) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) { + if blockNr, ok := blockNrOrHash.Number(); ok { + return b.StateAndHeaderByNumber(ctx, blockNr) + } + if hash, ok := blockNrOrHash.Hash(); ok { + header := b.eth.blockchain.GetHeaderByHash(hash) + if header == nil { + return nil, nil, errors.New("header for hash not found") + } + if blockNrOrHash.RequireCanonical && core.GetCanonicalHash(b.ChainDb(), header.Number.Uint64()) != hash { + return nil, nil, errors.New("hash is not currently canonical") + } + return light.NewState(ctx, header, b.eth.odr), header, nil + } + return nil, nil, errors.New("invalid arguments; neither block nor hash specified") +} + func (b *LesApiBackend) GetBlock(ctx context.Context, blockHash common.Hash) (*types.Block, error) { return b.eth.blockchain.GetBlockByHash(ctx, blockHash) } diff --git a/rpc/types.go b/rpc/types.go index f32f86bddc..58ed6c6982 100644 --- a/rpc/types.go +++ b/rpc/types.go @@ -17,13 +17,16 @@ package rpc import ( + "encoding/json" "fmt" "math" "reflect" + "strconv" "strings" "sync" mapset "github.com/deckarep/golang-set" + "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/common/hexutil" ) @@ -120,10 +123,12 @@ type BlockNumber int64 type EpochNumber int64 const ( - PendingBlockNumber = BlockNumber(-2) - LatestBlockNumber = BlockNumber(-1) - EarliestBlockNumber = BlockNumber(0) - LatestEpochNumber = EpochNumber(-1) + SafeBlockNumber = BlockNumber(-4) + FinalizedBlockNumber = BlockNumber(-3) + PendingBlockNumber = BlockNumber(-2) + LatestBlockNumber = BlockNumber(-1) + EarliestBlockNumber = BlockNumber(0) + LatestEpochNumber = EpochNumber(-1) ) // UnmarshalJSON parses the given JSON fragment into a BlockNumber. It supports: @@ -192,3 +197,112 @@ func trimData(data []byte) string { } return input } + +type BlockNumberOrHash struct { + BlockNumber *BlockNumber `json:"blockNumber,omitempty"` + BlockHash *common.Hash `json:"blockHash,omitempty"` + RequireCanonical bool `json:"requireCanonical,omitempty"` +} + +func (bnh *BlockNumberOrHash) UnmarshalJSON(data []byte) error { + type erased BlockNumberOrHash + e := erased{} + err := json.Unmarshal(data, &e) + if err == nil { + if e.BlockNumber != nil && e.BlockHash != nil { + return fmt.Errorf("cannot specify both BlockHash and BlockNumber, choose one or the other") + } + bnh.BlockNumber = e.BlockNumber + bnh.BlockHash = e.BlockHash + bnh.RequireCanonical = e.RequireCanonical + return nil + } + var input string + err = json.Unmarshal(data, &input) + if err != nil { + return err + } + switch input { + case "earliest": + bn := EarliestBlockNumber + bnh.BlockNumber = &bn + return nil + case "latest": + bn := LatestBlockNumber + bnh.BlockNumber = &bn + return nil + case "pending": + bn := PendingBlockNumber + bnh.BlockNumber = &bn + return nil + case "finalized": + bn := FinalizedBlockNumber + bnh.BlockNumber = &bn + return nil + case "safe": + bn := SafeBlockNumber + bnh.BlockNumber = &bn + return nil + default: + if len(input) == 66 { + hash := common.Hash{} + err := hash.UnmarshalText([]byte(input)) + if err != nil { + return err + } + bnh.BlockHash = &hash + return nil + } else { + blckNum, err := hexutil.DecodeUint64(input) + if err != nil { + return err + } + if blckNum > math.MaxInt64 { + return fmt.Errorf("blocknumber too high") + } + bn := BlockNumber(blckNum) + bnh.BlockNumber = &bn + return nil + } + } +} + +func (bnh *BlockNumberOrHash) Number() (BlockNumber, bool) { + if bnh.BlockNumber != nil { + return *bnh.BlockNumber, true + } + return BlockNumber(0), false +} + +func (bnh *BlockNumberOrHash) String() string { + if bnh.BlockNumber != nil { + return strconv.Itoa(int(*bnh.BlockNumber)) + } + if bnh.BlockHash != nil { + return bnh.BlockHash.String() + } + return "nil" +} + +func (bnh *BlockNumberOrHash) Hash() (common.Hash, bool) { + if bnh.BlockHash != nil { + return *bnh.BlockHash, true + } + return common.Hash{}, false +} + +func BlockNumberOrHashWithNumber(blockNr BlockNumber) BlockNumberOrHash { + return BlockNumberOrHash{ + BlockNumber: &blockNr, + BlockHash: nil, + RequireCanonical: false, + } +} + +func BlockNumberOrHashWithHash(hash common.Hash, canonical bool) BlockNumberOrHash { + return BlockNumberOrHash{ + BlockNumber: nil, + BlockHash: &hash, + RequireCanonical: canonical, + } +} From 7d492e2757d67710c6238d31c9183f3f222f875c Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Tue, 4 Jul 2023 13:37:23 +0700 Subject: [PATCH 43/82] [WIP] Fix sorting txs by nonce and price --- common/types.go | 17 +++++++++++++++-- contracts/utils.go | 9 ++++++++- core/state_transition.go | 4 ++-- core/types/transaction.go | 40 ++++++++++++++++++++++++--------------- internal/ethapi/api.go | 16 +++++++++++----- miner/worker.go | 2 +- 6 files changed, 62 insertions(+), 26 deletions(-) diff --git a/common/types.go b/common/types.go index ca94230113..eb377f7779 100644 --- a/common/types.go +++ b/common/types.go @@ -51,8 +51,21 @@ const ( ) var ( - hashT = reflect.TypeOf(Hash{}) - addressT = reflect.TypeOf(Address{}) + hashT = reflect.TypeOf(Hash{}) + addressT = reflect.TypeOf(Address{}) + SpecialSMCAddressesMap = map[string]bool{ + BlockSigners: true, + MasternodeVotingSMC: true, + RandomizeSMC: true, + FoudationAddr: true, + TeamAddr: true, + TomoXAddr: true, + TradingStateAddr: true, + TomoXLendingAddress: true, + TomoXLendingFinalizedTradeAddress: true, + TomoNativeAddress: true, + LendingLockAddress: true, + } ) // Hash represents the 32 byte Keccak256 hash of arbitrary data. diff --git a/contracts/utils.go b/contracts/utils.go index 4468b5de9a..ca83126117 100644 --- a/contracts/utils.go +++ b/contracts/utils.go @@ -169,7 +169,14 @@ func CreateTxSign(blockNumber *big.Int, blockHash common.Hash, nonce uint64, blo data := common.Hex2Bytes(common.HexSignMethod) inputData := append(data, common.LeftPadBytes(blockNumber.Bytes(), 32)...) inputData = append(inputData, common.LeftPadBytes(blockHash.Bytes(), 32)...) - tx := types.NewTransaction(nonce, blockSigner, big.NewInt(0), 200000, big.NewInt(0), inputData) + tx := types.NewTx(&types.LegacyTx{ + Nonce: nonce, + To: &blockSigner, + Value: big.NewInt(0), + Gas: 200000, + GasPrice: big.NewInt(0), + Data: inputData, + }) return tx } diff --git a/core/state_transition.go b/core/state_transition.go index 79463fe93c..5ea25beea4 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -274,9 +274,9 @@ func (st *StateTransition) preCheck() error { return fmt.Errorf("%w: address %v, maxPriorityFeePerGas: %s, maxFeePerGas: %s", ErrTipAboveFeeCap, msg.From.Hex(), msg.GasTipCap, msg.GasFeeCap) } - // This will panic if baseFee is nil, but basefee presence is verified + // This will panic if baseFee is nil, but BaseFee presence is verified // as part of header validation. - if msg.GasFeeCap.Cmp(st.evm.Context.BaseFee) < 0 { + if msg.GasFeeCap.Cmp(st.evm.Context.BaseFee) < 0 && !common.SpecialSMCAddressesMap[st.to().Address().String()] { return fmt.Errorf("%w: address %v, maxFeePerGas: %s baseFee: %s", ErrFeeCapTooLow, msg.From.Hex(), msg.GasFeeCap, st.evm.Context.BaseFee) } diff --git a/core/types/transaction.go b/core/types/transaction.go index 98f22e18be..e2856625d1 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -835,22 +835,32 @@ func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address]Transa delete(txs, from) continue } - lastSpecialTx := -1 - if len(signers) > 0 { - if _, ok := signers[from]; ok { - for i, tx := range accTxs { - if tx.IsSpecialTransaction() { - lastSpecialTx = i - } - } + //lastSpecialTx := -1 + //if len(signers) > 0 { + // if _, ok := signers[from]; ok { + // for i, tx := range accTxs { + // if tx.IsSpecialTransaction() { + // lastSpecialTx = i + // } + // } + // } + //} + //if lastSpecialTx >= 0 { + // for i := 0; i <= lastSpecialTx; i++ { + // specialTxs = append(specialTxs, accTxs[i]) + // } + //} else { + // heads.Push(wrapped) + // txs[from] = accTxs[1:] + //} + for _, tx := range accTxs { + if tx.IsSpecialTransaction() { + specialTxs = append(specialTxs, tx) + fmt.Printf("@@@@@@@@@@@@@@@@@@@@ append special tx %v\n", tx.Hash().Hex()) + } else { + heads.Push(wrapped) + fmt.Printf("@@@@@@@@@@@@@@@@@@@@ append normal tx %v\n", tx.Hash().Hex()) } - } - if lastSpecialTx >= 0 { - for i := 0; i <= lastSpecialTx; i++ { - specialTxs = append(specialTxs, accTxs[i]) - } - } else { - heads.Push(wrapped) txs[from] = accTxs[1:] } } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 8af445fe11..e0ffdd0067 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1141,6 +1141,9 @@ func (s *PublicBlockChainAPI) EstimateGas(ctx context.Context, args TransactionA if err != nil { return 0, err } + if block == nil { + return 0, errors.New("block not found") + } hi = block.GasLimit() } @@ -1822,6 +1825,10 @@ func (s *PublicTransactionPoolAPI) GetTransactionReceipt(ctx context.Context, ha if tx == nil { return nil, nil } + header, err := s.b.HeaderByHash(ctx, blockHash) + if err != nil { + return nil, err + } receipts, err := s.b.GetReceipts(ctx, blockHash) if err != nil { return nil, err @@ -1831,10 +1838,8 @@ func (s *PublicTransactionPoolAPI) GetTransactionReceipt(ctx context.Context, ha } receipt := receipts[index] - var signer types.Signer = types.FrontierSigner{} - if tx.Protected() { - signer = types.NewEIP155Signer(tx.ChainId()) - } + // Derive the sender. + signer := types.MakeSigner(s.b.ChainConfig(), header.Number) from, _ := types.Sender(signer, tx) fields := map[string]interface{}{ @@ -1849,6 +1854,8 @@ func (s *PublicTransactionPoolAPI) GetTransactionReceipt(ctx context.Context, ha "contractAddress": nil, "logs": receipt.Logs, "logsBloom": receipt.Bloom, + "type": hexutil.Uint(tx.Type()), + "effectiveGasPrice": (*hexutil.Big)(receipt.EffectiveGasPrice), } // Assign receipt status or post state. @@ -2250,7 +2257,6 @@ func (s *PublicTransactionPoolAPI) SendRawTransaction(ctx context.Context, encod if err := tx.UnmarshalBinary(encodedTx); err != nil { return common.Hash{}, err } - fmt.Printf("@@@@@@@@@@@@@@@@@@@@@@@@@@@@ SendRawTransaction: %+v\n", tx.Hash().Hex()) return submitTransaction(ctx, s.b, tx) } diff --git a/miner/worker.go b/miner/worker.go index c354d78cb7..3d9b75189b 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -474,7 +474,7 @@ func (self *worker) makeCurrent(parent *types.Block, header *types.Header) error work := &Work{ config: self.config, - signer: types.NewEIP155Signer(self.config.ChainId), + signer: types.MakeSigner(self.config, header.Number), state: state, parentState: state.Copy(), tradingState: tomoxState, From 1b1f5f8fa22eaf4e716a0bd198712fb85342aab9 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Tue, 4 Jul 2023 15:11:08 +0700 Subject: [PATCH 44/82] Fix some RPC APIs to adapt Hardhat --- core/state/access_list.go | 136 ++++++++++++++++++++++++++++++++++++++ core/state/journal.go | 33 +++++++++ core/state/statedb.go | 50 +++++++++++++- core/state_processor.go | 1 - core/types/transaction.go | 5 +- core/vm/errors.go | 1 + core/vm/evm.go | 26 +++++--- core/vm/interface.go | 2 + internal/ethapi/api.go | 10 ++- 9 files changed, 248 insertions(+), 16 deletions(-) create mode 100644 core/state/access_list.go diff --git a/core/state/access_list.go b/core/state/access_list.go new file mode 100644 index 0000000000..e916f29334 --- /dev/null +++ b/core/state/access_list.go @@ -0,0 +1,136 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "github.com/tomochain/tomochain/common" +) + +type accessList struct { + addresses map[common.Address]int + slots []map[common.Hash]struct{} +} + +// ContainsAddress returns true if the address is in the access list. +func (al *accessList) ContainsAddress(address common.Address) bool { + _, ok := al.addresses[address] + return ok +} + +// Contains checks if a slot within an account is present in the access list, returning +// separate flags for the presence of the account and the slot respectively. +func (al *accessList) Contains(address common.Address, slot common.Hash) (addressPresent bool, slotPresent bool) { + idx, ok := al.addresses[address] + if !ok { + // no such address (and hence zero slots) + return false, false + } + if idx == -1 { + // address yes, but no slots + return true, false + } + _, slotPresent = al.slots[idx][slot] + return true, slotPresent +} + +// newAccessList creates a new accessList. +func newAccessList() *accessList { + return &accessList{ + addresses: make(map[common.Address]int), + } +} + +// Copy creates an independent copy of an accessList. +func (a *accessList) Copy() *accessList { + cp := newAccessList() + for k, v := range a.addresses { + cp.addresses[k] = v + } + cp.slots = make([]map[common.Hash]struct{}, len(a.slots)) + for i, slotMap := range a.slots { + newSlotmap := make(map[common.Hash]struct{}, len(slotMap)) + for k := range slotMap { + newSlotmap[k] = struct{}{} + } + cp.slots[i] = newSlotmap + } + return cp +} + +// AddAddress adds an address to the access list, and returns 'true' if the operation +// caused a change (addr was not previously in the list). +func (al *accessList) AddAddress(address common.Address) bool { + if _, present := al.addresses[address]; present { + return false + } + al.addresses[address] = -1 + return true +} + +// AddSlot adds the specified (addr, slot) combo to the access list. +// Return values are: +// - address added +// - slot added +// For any 'true' value returned, a corresponding journal entry must be made. +func (al *accessList) AddSlot(address common.Address, slot common.Hash) (addrChange bool, slotChange bool) { + idx, addrPresent := al.addresses[address] + if !addrPresent || idx == -1 { + // Address not present, or addr present but no slots there + al.addresses[address] = len(al.slots) + slotmap := map[common.Hash]struct{}{slot: {}} + al.slots = append(al.slots, slotmap) + return !addrPresent, true + } + // There is already an (address,slot) mapping + slotmap := al.slots[idx] + if _, ok := slotmap[slot]; !ok { + slotmap[slot] = struct{}{} + // Journal add slot change + return false, true + } + // No changes required + return false, false +} + +// DeleteSlot removes an (address, slot)-tuple from the access list. +// This operation needs to be performed in the same order as the addition happened. +// This method is meant to be used by the journal, which maintains ordering of +// operations. +func (al *accessList) DeleteSlot(address common.Address, slot common.Hash) { + idx, addrOk := al.addresses[address] + // There are two ways this can fail + if !addrOk { + panic("reverting slot change, address not present in list") + } + slotmap := al.slots[idx] + delete(slotmap, slot) + // If that was the last (first) slot, remove it + // Since additions and rollbacks are always performed in order, + // we can delete the item without worrying about screwing up later indices + if len(slotmap) == 0 { + al.slots = al.slots[:idx] + al.addresses[address] = -1 + } +} + +// DeleteAddress removes an address from the access list. This operation +// needs to be performed in the same order as the addition happened. +// This method is meant to be used by the journal, which maintains ordering of +// operations. +func (al *accessList) DeleteAddress(address common.Address) { + delete(al.addresses, address) +} diff --git a/core/state/journal.go b/core/state/journal.go index 1ac5cdbf25..34324237f1 100644 --- a/core/state/journal.go +++ b/core/state/journal.go @@ -75,6 +75,14 @@ type ( prev bool prevDirty bool } + // Changes to the access list + accessListAddAccountChange struct { + address *common.Address + } + accessListAddSlotChange struct { + address *common.Address + slot *common.Hash + } ) func (ch createObjectChange) undo(s *StateDB) { @@ -138,3 +146,28 @@ func (ch addLogChange) undo(s *StateDB) { func (ch addPreimageChange) undo(s *StateDB) { delete(s.preimages, ch.hash) } + +func (ch accessListAddAccountChange) undo(s *StateDB) { + /* + One important invariant here, is that whenever a (addr, slot) is added, if the + addr is not already present, the add causes two journal entries: + - one for the address, + - one for the (address,slot) + Therefore, when unrolling the change, we can always blindly delete the + (addr) at this point, since no storage adds can remain when come upon + a single (addr) change. + */ + s.accessList.DeleteAddress(*ch.address) +} + +func (ch accessListAddAccountChange) dirtied() *common.Address { + return nil +} + +func (ch accessListAddSlotChange) undo(s *StateDB) { + s.accessList.DeleteSlot(*ch.address, *ch.slot) +} + +func (ch accessListAddSlotChange) dirtied() *common.Address { + return nil +} diff --git a/core/state/statedb.go b/core/state/statedb.go index 7a3357b3e8..d5491a87d2 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -74,6 +74,9 @@ type StateDB struct { preimages map[common.Hash][]byte + // Per-transaction access list + accessList *accessList + // Journal of state modifications. This is the backbone of // Snapshot and RevertToSnapshot. journal journal @@ -113,6 +116,7 @@ func New(root common.Hash, db Database) (*StateDB, error) { stateObjectsDirty: make(map[common.Address]struct{}), logs: make(map[common.Hash][]*types.Log), preimages: make(map[common.Hash][]byte), + accessList: newAccessList(), }, nil } @@ -449,8 +453,8 @@ func (self *StateDB) createObject(addr common.Address) (newobj, prev *stateObjec // CreateAccount is called during the EVM CREATE operation. The situation might arise that // a contract does the following: // -// 1. sends funds to sha(account ++ (nonce + 1)) -// 2. tx_create(sha(account ++ nonce)) (note that this gets the address of 1) +// 1. sends funds to sha(account ++ (nonce + 1)) +// 2. tx_create(sha(account ++ nonce)) (note that this gets the address of 1) // // Carrying over the balance ensures that Ether doesn't disappear. func (self *StateDB) CreateAccount(addr common.Address) { @@ -511,6 +515,13 @@ func (self *StateDB) Copy() *StateDB { for hash, preimage := range self.preimages { state.preimages[hash] = preimage } + // Do we need to copy the access list and transient storage? + // In practice: No. At the start of a transaction, these two lists are empty. + // In practice, we only ever copy state _between_ transactions/blocks, never + // in the middle of a transaction. However, it doesn't cost us much to copy + // empty lists, so we do it anyway to not blow up if we ever decide copy them + // in the middle of a transaction. + state.accessList = self.accessList.Copy() return state } @@ -580,6 +591,41 @@ func (self *StateDB) Prepare(thash, bhash common.Hash, ti int) { self.txIndex = ti } +// AddAddressToAccessList adds the given address to the access list +func (s *StateDB) AddAddressToAccessList(addr common.Address) { + if s.accessList.AddAddress(addr) { + s.journal = append(s.journal, accessListAddAccountChange{&addr}) + } +} + +// AddSlotToAccessList adds the given (address, slot)-tuple to the access list +func (s *StateDB) AddSlotToAccessList(addr common.Address, slot common.Hash) { + addrMod, slotMod := s.accessList.AddSlot(addr, slot) + if addrMod { + // In practice, this should not happen, since there is no way to enter the + // scope of 'address' without having the 'address' become already added + // to the access list (via call-variant, create, etc). + // Better safe than sorry, though + s.journal = append(s.journal, accessListAddAccountChange{&addr}) + } + if slotMod { + s.journal = append(s.journal, accessListAddSlotChange{ + address: &addr, + slot: &slot, + }) + } +} + +// AddressInAccessList returns true if the given address is in the access list. +func (s *StateDB) AddressInAccessList(addr common.Address) bool { + return s.accessList.ContainsAddress(addr) +} + +// SlotInAccessList returns true if the given (address, slot)-tuple is in the access list. +func (s *StateDB) SlotInAccessList(addr common.Address, slot common.Hash) (addressPresent bool, slotPresent bool) { + return s.accessList.Contains(addr, slot) +} + // DeleteSuicides flags the suicided objects for deletion so that it // won't be referenced again when called / queried up on. // diff --git a/core/state_processor.go b/core/state_processor.go index 1bb5044369..77e894330b 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -409,7 +409,6 @@ func ApplyTransaction(config *params.ChainConfig, tokensFee map[common.Address]* // Apply the transaction to the current state (included in the env) _, gas, failed, err := ApplyMessage(vmenv, msg, gp, coinbaseOwner) - if err != nil { return nil, 0, err, false } diff --git a/core/types/transaction.go b/core/types/transaction.go index e2856625d1..55d52f40b2 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -831,7 +831,7 @@ func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address]Transa acc, _ := Sender(signer, accTxs[0]) wrapped, err := NewTxWithMinerFee(accTxs[0], baseFee) // Remove transaction if sender doesn't match from, or if wrapping fails. - if acc != from || err != nil { + if (acc != from || err != nil) && !accTxs[0].IsSpecialTransaction() { delete(txs, from) continue } @@ -853,13 +853,12 @@ func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address]Transa // heads.Push(wrapped) // txs[from] = accTxs[1:] //} + // TODO(trinhdn2): remove the for loop for _, tx := range accTxs { if tx.IsSpecialTransaction() { specialTxs = append(specialTxs, tx) - fmt.Printf("@@@@@@@@@@@@@@@@@@@@ append special tx %v\n", tx.Hash().Hex()) } else { heads.Push(wrapped) - fmt.Printf("@@@@@@@@@@@@@@@@@@@@ append normal tx %v\n", tx.Hash().Hex()) } txs[from] = accTxs[1:] } diff --git a/core/vm/errors.go b/core/vm/errors.go index c813aa36af..c7cfeae53c 100644 --- a/core/vm/errors.go +++ b/core/vm/errors.go @@ -34,6 +34,7 @@ var ( ErrWriteProtection = errors.New("write protection") ErrReturnDataOutOfBounds = errors.New("return data out of bounds") ErrGasUintOverflow = errors.New("gas uint64 overflow") + ErrInvalidCode = errors.New("invalid code: must not begin with 0xef") ) // ErrStackUnderflow wraps an evm error when the items on the stack less diff --git a/core/vm/evm.go b/core/vm/evm.go index 88654455ec..fbf7f289ab 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -394,6 +394,12 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, nonce := evm.StateDB.GetNonce(caller.Address()) evm.StateDB.SetNonce(caller.Address(), nonce+1) + // We add this to the access list _before_ taking a snapshot. Even if the creation fails, + // the access-list change should not be rolled back + if evm.chainRules.IsLondon { + evm.StateDB.AddAddressToAccessList(address) + } + // Ensure there's no existing contract already at the designated address contractHash := evm.StateDB.GetCodeHash(address) if evm.StateDB.GetNonce(address) != 0 || (contractHash != (common.Hash{}) && contractHash != emptyCodeHash) { @@ -423,13 +429,20 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, ret, err := run(evm, contract, nil, false) - // check whether the max code size has been exceeded - maxCodeSizeExceeded := evm.chainRules.IsEIP158 && len(ret) > params.MaxCodeSize + // Check whether the max code size has been exceeded, assign err if the case. + if err == nil && evm.chainRules.IsEIP158 && len(ret) > params.MaxCodeSize { + err = ErrMaxCodeSizeExceeded + } + // Reject code starting with 0xEF if EIP-3541 is enabled. + if err == nil && len(ret) >= 1 && ret[0] == 0xEF && evm.chainRules.IsLondon { + err = ErrInvalidCode + } + // if the contract creation ran successfully and no errors were returned // calculate the gas required to store the code. If the code could not // be stored due to not enough gas set an error and let it be handled // by the error checking condition below. - if err == nil && !maxCodeSizeExceeded { + if err == nil { createDataGas := uint64(len(ret)) * params.CreateDataGas if contract.UseGas(createDataGas) { evm.StateDB.SetCode(address, ret) @@ -441,21 +454,16 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, // When an error was returned by the EVM or when setting the creation code // above we revert to the snapshot and consume any gas remaining. Additionally // when we're in homestead this also counts for code storage gas errors. - if maxCodeSizeExceeded || (err != nil && (evm.chainRules.IsHomestead || err != ErrCodeStoreOutOfGas)) { + if err != nil && (evm.chainRules.IsHomestead || err != ErrCodeStoreOutOfGas) { evm.StateDB.RevertToSnapshot(snapshot) if err != ErrExecutionReverted { contract.UseGas(contract.Gas) } } - // Assign err if contract code size exceeds the max while the err is still empty. - if maxCodeSizeExceeded && err == nil { - err = ErrMaxCodeSizeExceeded - } if evm.Config.Debug && evm.depth == 0 { evm.Config.Tracer.CaptureEnd(ret, gas-contract.Gas, time.Since(start), err) } return ret, address, contract.Gas, err - } // Create creates a new contract using code as deployment code. diff --git a/core/vm/interface.go b/core/vm/interface.go index 6df0dede26..cd25394fe6 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -64,6 +64,8 @@ type StateDB interface { AddPreimage(common.Hash, []byte) ForEachStorage(common.Address, func(common.Hash, common.Hash) bool) error + + AddAddressToAccessList(common.Address) } // CallContext provides a basic interface for the EVM calling conventions. The EVM diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index e0ffdd0067..1e201aaeab 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -87,6 +87,9 @@ func (s *PublicEthereumAPI) GasPrice(ctx context.Context) (*big.Int, error) { if head := s.b.CurrentHeader(); head.BaseFee != nil { tipcap.Add(tipcap, head.BaseFee) } + if tipcap.Cmp(common.MinGasPrice) < 0 { + tipcap = common.MinGasPrice + } return tipcap, err } @@ -1145,6 +1148,9 @@ func (s *PublicBlockChainAPI) EstimateGas(ctx context.Context, args TransactionA return 0, errors.New("block not found") } hi = block.GasLimit() + if s.b.ChainConfig().IsLondon(block.Header().Number) && args.MaxFeePerGas == nil && args.GasPrice == nil { + args.MaxFeePerGas = (*hexutil.Big)(block.BaseFee()) + } } // Normalize the max fee per gas the call is willing to spend. @@ -1779,7 +1785,9 @@ func (s *PublicTransactionPoolAPI) GetRawTransactionByBlockHashAndIndex(ctx cont func (s *PublicTransactionPoolAPI) GetTransactionCount(ctx context.Context, address common.Address, blockNr rpc.BlockNumber) (*hexutil.Uint64, error) { state, _, err := s.b.StateAndHeaderByNumber(ctx, blockNr) if state == nil || err != nil { - return nil, err + if state, _, err = s.b.StateAndHeaderByNumber(ctx, rpc.LatestBlockNumber); state == nil || err != nil { + return nil, err + } } nonce := state.GetNonce(address) return (*hexutil.Uint64)(&nonce), state.Error() From f659ea0c6dce7cd8464da4662bfcd7f7b2ce6450 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Tue, 4 Jul 2023 18:11:47 +0700 Subject: [PATCH 45/82] Fix eth_estimate Gas API --- common/types.go | 17 ++--------------- core/state_transition.go | 2 +- internal/ethapi/api.go | 11 ++++------- 3 files changed, 7 insertions(+), 23 deletions(-) diff --git a/common/types.go b/common/types.go index eb377f7779..ca94230113 100644 --- a/common/types.go +++ b/common/types.go @@ -51,21 +51,8 @@ const ( ) var ( - hashT = reflect.TypeOf(Hash{}) - addressT = reflect.TypeOf(Address{}) - SpecialSMCAddressesMap = map[string]bool{ - BlockSigners: true, - MasternodeVotingSMC: true, - RandomizeSMC: true, - FoudationAddr: true, - TeamAddr: true, - TomoXAddr: true, - TradingStateAddr: true, - TomoXLendingAddress: true, - TomoXLendingFinalizedTradeAddress: true, - TomoNativeAddress: true, - LendingLockAddress: true, - } + hashT = reflect.TypeOf(Hash{}) + addressT = reflect.TypeOf(Address{}) ) // Hash represents the 32 byte Keccak256 hash of arbitrary data. diff --git a/core/state_transition.go b/core/state_transition.go index 5ea25beea4..35c4e61c92 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -276,7 +276,7 @@ func (st *StateTransition) preCheck() error { } // This will panic if baseFee is nil, but BaseFee presence is verified // as part of header validation. - if msg.GasFeeCap.Cmp(st.evm.Context.BaseFee) < 0 && !common.SpecialSMCAddressesMap[st.to().Address().String()] { + if msg.GasFeeCap.Cmp(st.evm.Context.BaseFee) < 0 { return fmt.Errorf("%w: address %v, maxFeePerGas: %s baseFee: %s", ErrFeeCapTooLow, msg.From.Hex(), msg.GasFeeCap, st.evm.Context.BaseFee) } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 1e201aaeab..32f7602a0b 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1035,7 +1035,7 @@ func (s *PublicBlockChainAPI) getCandidatesFromSmartContract() ([]posv.Masternod return candidatesWithStakeInfo, nil } -func (s *PublicBlockChainAPI) doCall(ctx context.Context, args TransactionArgs, blockNr rpc.BlockNumber, vmCfg vm.Config, timeout time.Duration) ([]byte, uint64, bool, error) { +func (s *PublicBlockChainAPI) doCall(ctx context.Context, args TransactionArgs, blockNr rpc.BlockNumber, timeout time.Duration) ([]byte, uint64, bool, error) { defer func(start time.Time) { log.Debug("Executing EVM call finished", "runtime", time.Since(start)) }(time.Now()) statedb, header, err := s.b.StateAndHeaderByNumber(ctx, blockNr) @@ -1093,7 +1093,7 @@ func (s *PublicBlockChainAPI) doCall(ctx context.Context, args TransactionArgs, return nil, 0, false, err } // Get a new instance of the EVM. - evm, vmError, err := s.b.GetEVM(ctx, msg, statedb, tomoxState, header, vmCfg) + evm, vmError, err := s.b.GetEVM(ctx, msg, statedb, tomoxState, header, vm.Config{NoBaseFee: true}) if err != nil { return nil, 0, false, err } @@ -1118,7 +1118,7 @@ func (s *PublicBlockChainAPI) doCall(ctx context.Context, args TransactionArgs, // Call executes the given transaction on the state for the given block number. // It doesn't make and changes in the state/blockchain and is useful to execute and retrieve values. func (s *PublicBlockChainAPI) Call(ctx context.Context, args TransactionArgs, blockNr rpc.BlockNumber) (hexutil.Bytes, error) { - result, _, _, err := s.doCall(ctx, args, blockNr, vm.Config{}, 5*time.Second) + result, _, _, err := s.doCall(ctx, args, blockNr, 5*time.Second) return (hexutil.Bytes)(result), err } @@ -1148,9 +1148,6 @@ func (s *PublicBlockChainAPI) EstimateGas(ctx context.Context, args TransactionA return 0, errors.New("block not found") } hi = block.GasLimit() - if s.b.ChainConfig().IsLondon(block.Header().Number) && args.MaxFeePerGas == nil && args.GasPrice == nil { - args.MaxFeePerGas = (*hexutil.Big)(block.BaseFee()) - } } // Normalize the max fee per gas the call is willing to spend. @@ -1197,7 +1194,7 @@ func (s *PublicBlockChainAPI) EstimateGas(ctx context.Context, args TransactionA executable := func(gas uint64) bool { args.Gas = (*hexutil.Uint64)(&gas) - _, _, failed, err := s.doCall(ctx, args, rpc.LatestBlockNumber, vm.Config{}, 0) + _, _, failed, err := s.doCall(ctx, args, rpc.LatestBlockNumber, 0) if err != nil || failed { return false } From 58f1d13f86738eb91b7e8e1055e860fc1345e136 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Tue, 4 Jul 2023 18:49:02 +0700 Subject: [PATCH 46/82] Derive more external field for receipts --- accounts/abi/bind/backends/simulated.go | 11 +++++++---- contracts/utils.go | 2 +- core/blockchain.go | 4 ++-- core/database_util.go | 26 +++++++++++++++++++++---- eth/api_backend.go | 6 +++--- eth/downloader/fakepeer.go | 2 +- eth/filters/filter.go | 6 ++++-- eth/filters/filter_system.go | 2 +- internal/ethapi/api.go | 2 +- internal/ethapi/backend.go | 2 +- les/api_backend.go | 6 +++--- les/handler.go | 4 ++-- light/odr_util.go | 9 +++++---- light/txpool.go | 2 +- 14 files changed, 54 insertions(+), 30 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 6da52e0752..1b55704812 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -175,7 +175,7 @@ func (b *SimulatedBackend) ForEachStorageAt(ctx context.Context, contract common // TransactionReceipt returns the receipt of a transaction. func (b *SimulatedBackend) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) { - receipt, _, _, _ := core.GetReceipt(b.database, txHash) + receipt, _, _, _ := core.GetReceipt(b.database, txHash, b.config) return receipt, nil } @@ -569,12 +569,12 @@ func (fb *filterBackend) HeaderByNumber(ctx context.Context, block rpc.BlockNumb return fb.bc.GetHeaderByNumber(uint64(block.Int64())), nil } -func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { - return core.GetBlockReceipts(fb.db, hash, core.GetBlockNumber(fb.db, hash)), nil +func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash, config *params.ChainConfig) (types.Receipts, error) { + return core.GetBlockReceipts(fb.db, hash, core.GetBlockNumber(fb.db, hash), config), nil } func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { - receipts := core.GetBlockReceipts(fb.db, hash, core.GetBlockNumber(fb.db, hash)) + receipts := core.GetBlockReceipts(fb.db, hash, core.GetBlockNumber(fb.db, hash), fb.bc.Config()) if receipts == nil { return nil, nil } @@ -605,3 +605,6 @@ func (fb *filterBackend) BloomStatus() (uint64, uint64) { return 4096, 0 } func (fb *filterBackend) ServiceFilter(ctx context.Context, ms *bloombits.MatcherSession) { panic("not supported") } +func (fb *filterBackend) ChainConfig() *params.ChainConfig { + panic("not supported") +} diff --git a/contracts/utils.go b/contracts/utils.go index ca83126117..551e0be9cd 100644 --- a/contracts/utils.go +++ b/contracts/utils.go @@ -343,7 +343,7 @@ func GetRewardForCheckpoint(c *posv.Posv, chain consensus.ChainReader, header *t block := chain.GetBlock(header.Hash(), i) txs := block.Transactions() if !chain.Config().IsTIPSigning(header.Number) { - receipts := core.GetBlockReceipts(c.GetDb(), header.Hash(), i) + receipts := core.GetBlockReceipts(c.GetDb(), header.Hash(), i, chain.Config()) signData = c.CacheData(header, txs, receipts) } else { signData = c.CacheSigner(header.Hash(), txs) diff --git a/core/blockchain.go b/core/blockchain.go index f763189be7..27c5e0554b 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -800,7 +800,7 @@ func (bc *BlockChain) GetBlockByNumber(number uint64) *types.Block { // GetReceiptsByHash retrieves the receipts for all transactions in a given block. func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts { - return GetBlockReceipts(bc.db, hash, GetBlockNumber(bc.db, hash)) + return GetBlockReceipts(bc.db, hash, GetBlockNumber(bc.db, hash), bc.chainConfig) } // GetBlocksFromHash returns the block corresponding to hash and up to n-1 ancestors. @@ -2120,7 +2120,7 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // These logs are later announced as deleted. collectLogs = func(h common.Hash) { // Coalesce logs and set 'Removed'. - receipts := GetBlockReceipts(bc.db, h, bc.hc.GetBlockNumber(h)) + receipts := GetBlockReceipts(bc.db, h, bc.hc.GetBlockNumber(h), bc.chainConfig) for _, receipt := range receipts { for _, log := range receipt.Logs { del := *log diff --git a/core/database_util.go b/core/database_util.go index a5ab18687d..2afce9a124 100644 --- a/core/database_util.go +++ b/core/database_util.go @@ -22,10 +22,10 @@ import ( "encoding/json" "errors" "fmt" - "github.com/tomochain/tomochain/core/rawdb" "math/big" "github.com/tomochain/tomochain/common" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/ethdb" "github.com/tomochain/tomochain/log" @@ -246,7 +246,7 @@ func GetBlock(db DatabaseReader, hash common.Hash, number uint64) *types.Block { // GetBlockReceipts retrieves the receipts generated by the transactions included // in a block given by its hash. -func GetBlockReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Receipts { +func GetBlockReceipts(db DatabaseReader, hash common.Hash, number uint64, config *params.ChainConfig) types.Receipts { data, _ := db.Get(append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash[:]...)) if len(data) == 0 { return nil @@ -260,6 +260,24 @@ func GetBlockReceipts(db DatabaseReader, hash common.Hash, number uint64) types. for i, receipt := range storageReceipts { receipts[i] = (*types.Receipt)(receipt) } + + block := GetBlock(db, hash, number) + if block == nil { + log.Error("Missing block body but have receipt", "hash", hash, "number", number) + return nil + } + header := GetHeader(db, hash, number) + var baseFee *big.Int + if header == nil { + baseFee = big.NewInt(0) + } else { + baseFee = header.BaseFee + } + if err := receipts.DeriveFields(config, hash, number, baseFee, ([]*types.Transaction)(block.Transactions())); err != nil { + log.Error("Failed to derive block receipts fields", "hash", hash, "number", number, "err", err) + return nil + } + return receipts } @@ -317,12 +335,12 @@ func GetTransaction(db DatabaseReader, hash common.Hash) (*types.Transaction, co // GetReceipt retrieves a specific transaction receipt from the database, along with // its added positional metadata. -func GetReceipt(db DatabaseReader, hash common.Hash) (*types.Receipt, common.Hash, uint64, uint64) { +func GetReceipt(db DatabaseReader, hash common.Hash, config *params.ChainConfig) (*types.Receipt, common.Hash, uint64, uint64) { // Retrieve the lookup metadata and resolve the receipt from the receipts blockHash, blockNumber, receiptIndex := GetTxLookupEntry(db, hash) if blockHash != (common.Hash{}) { - receipts := GetBlockReceipts(db, blockHash, blockNumber) + receipts := GetBlockReceipts(db, blockHash, blockNumber, config) if len(receipts) <= int(receiptIndex) { log.Error("Receipt refereced missing", "number", blockNumber, "hash", blockHash, "index", receiptIndex) return nil, common.Hash{}, 0, 0 diff --git a/eth/api_backend.go b/eth/api_backend.go index 8b6a4826f5..4164a83867 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -159,12 +159,12 @@ func (b *EthApiBackend) GetBlock(ctx context.Context, blockHash common.Hash) (*t return b.eth.blockchain.GetBlockByHash(blockHash), nil } -func (b *EthApiBackend) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) { - return core.GetBlockReceipts(b.eth.chainDb, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash)), nil +func (b *EthApiBackend) GetReceipts(ctx context.Context, blockHash common.Hash, config *params.ChainConfig) (types.Receipts, error) { + return core.GetBlockReceipts(b.eth.chainDb, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash), config), nil } func (b *EthApiBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) { - receipts := core.GetBlockReceipts(b.eth.chainDb, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash)) + receipts := core.GetBlockReceipts(b.eth.chainDb, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash), b.ChainConfig()) if receipts == nil { return nil, nil } diff --git a/eth/downloader/fakepeer.go b/eth/downloader/fakepeer.go index 4d7c5ac280..c2a5178342 100644 --- a/eth/downloader/fakepeer.go +++ b/eth/downloader/fakepeer.go @@ -140,7 +140,7 @@ func (p *FakePeer) RequestBodies(hashes []common.Hash) error { func (p *FakePeer) RequestReceipts(hashes []common.Hash) error { var receipts [][]*types.Receipt for _, hash := range hashes { - receipts = append(receipts, core.GetBlockReceipts(p.db, hash, p.hc.GetBlockNumber(hash))) + receipts = append(receipts, core.GetBlockReceipts(p.db, hash, p.hc.GetBlockNumber(hash), p.hc.Config())) } p.dl.DeliverReceipts(p.id, receipts) return nil diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 534439074a..7f0ea75fe1 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -26,6 +26,7 @@ import ( "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/ethdb" "github.com/tomochain/tomochain/event" + "github.com/tomochain/tomochain/params" "github.com/tomochain/tomochain/rpc" ) @@ -33,8 +34,9 @@ type Backend interface { ChainDb() ethdb.Database EventMux() *event.TypeMux HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) - GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) + GetReceipts(ctx context.Context, blockHash common.Hash, config *params.ChainConfig) (types.Receipts, error) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) + ChainConfig() *params.ChainConfig SubscribeTxPreEvent(chan<- core.TxPreEvent) event.Subscription SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription @@ -214,7 +216,7 @@ func (f *Filter) checkMatches(ctx context.Context, header *types.Header) (logs [ if len(logs) > 0 { // We have matching logs, check if we need to resolve full logs via the light client if logs[0].TxHash == (common.Hash{}) { - receipts, err := f.backend.GetReceipts(ctx, header.Hash()) + receipts, err := f.backend.GetReceipts(ctx, header.Hash(), f.backend.ChainConfig()) if err != nil { return nil, err } diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index 3d92fc1ac7..4c15d29a88 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -390,7 +390,7 @@ func (es *EventSystem) lightFilterLogs(header *types.Header, addresses []common. logs := filterLogs(unfiltered, nil, nil, addresses, topics) if len(logs) > 0 && logs[0].TxHash == (common.Hash{}) { // We have matching but non-derived logs - receipts, err := es.backend.GetReceipts(ctx, header.Hash()) + receipts, err := es.backend.GetReceipts(ctx, header.Hash(), es.backend.ChainConfig()) if err != nil { return nil } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 32f7602a0b..e9ea6c4386 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1834,7 +1834,7 @@ func (s *PublicTransactionPoolAPI) GetTransactionReceipt(ctx context.Context, ha if err != nil { return nil, err } - receipts, err := s.b.GetReceipts(ctx, blockHash) + receipts, err := s.b.GetReceipts(ctx, blockHash, s.b.ChainConfig()) if err != nil { return nil, err } diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index a3461db669..5500e9d1fb 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -62,7 +62,7 @@ type Backend interface { StateAndHeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*state.StateDB, *types.Header, error) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) GetBlock(ctx context.Context, blockHash common.Hash) (*types.Block, error) - GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) + GetReceipts(ctx context.Context, blockHash common.Hash, config *params.ChainConfig) (types.Receipts, error) GetTd(blockHash common.Hash) *big.Int GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, tomoxState *tradingstate.TradingStateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription diff --git a/les/api_backend.go b/les/api_backend.go index f433b83ffc..3e5b035802 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -133,12 +133,12 @@ func (b *LesApiBackend) GetBlock(ctx context.Context, blockHash common.Hash) (*t return b.eth.blockchain.GetBlockByHash(ctx, blockHash) } -func (b *LesApiBackend) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) { - return light.GetBlockReceipts(ctx, b.eth.odr, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash)) +func (b *LesApiBackend) GetReceipts(ctx context.Context, blockHash common.Hash, config *params.ChainConfig) (types.Receipts, error) { + return light.GetBlockReceipts(ctx, b.eth.odr, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash), config) } func (b *LesApiBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) { - return light.GetBlockLogs(ctx, b.eth.odr, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash)) + return light.GetBlockLogs(ctx, b.eth.odr, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash), b.ChainConfig()) } func (b *LesApiBackend) GetTd(blockHash common.Hash) *big.Int { diff --git a/les/handler.go b/les/handler.go index b426f7fdd1..c338ca62ae 100644 --- a/les/handler.go +++ b/les/handler.go @@ -21,7 +21,6 @@ import ( "encoding/binary" "errors" "fmt" - "github.com/tomochain/tomochain/core/rawdb" "math/big" "net" "sync" @@ -30,6 +29,7 @@ import ( "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/consensus" "github.com/tomochain/tomochain/core" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/state" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/eth/downloader" @@ -646,7 +646,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { break } // Retrieve the requested block's receipts, skipping if unknown to us - results := core.GetBlockReceipts(pm.chainDb, hash, core.GetBlockNumber(pm.chainDb, hash)) + results := core.GetBlockReceipts(pm.chainDb, hash, core.GetBlockNumber(pm.chainDb, hash), pm.chainConfig) if results == nil { if header := pm.blockchain.GetHeaderByHash(hash); header == nil || header.ReceiptHash != types.EmptyRootHash { continue diff --git a/light/odr_util.go b/light/odr_util.go index 89a63eb2b9..6adbc9303f 100644 --- a/light/odr_util.go +++ b/light/odr_util.go @@ -24,6 +24,7 @@ import ( "github.com/tomochain/tomochain/core" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/crypto" + "github.com/tomochain/tomochain/params" "github.com/tomochain/tomochain/rlp" ) @@ -125,9 +126,9 @@ func GetBlock(ctx context.Context, odr OdrBackend, hash common.Hash, number uint // GetBlockReceipts retrieves the receipts generated by the transactions included // in a block given by its hash. -func GetBlockReceipts(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) (types.Receipts, error) { +func GetBlockReceipts(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64, config *params.ChainConfig) (types.Receipts, error) { // Retrieve the potentially incomplete receipts from disk or network - receipts := core.GetBlockReceipts(odr.Database(), hash, number) + receipts := core.GetBlockReceipts(odr.Database(), hash, number, config) if receipts == nil { r := &ReceiptsRequest{Hash: hash, Number: number} if err := odr.Retrieve(ctx, r); err != nil { @@ -154,9 +155,9 @@ func GetBlockReceipts(ctx context.Context, odr OdrBackend, hash common.Hash, num // GetBlockLogs retrieves the logs generated by the transactions included in a // block given by its hash. -func GetBlockLogs(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) ([][]*types.Log, error) { +func GetBlockLogs(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64, config *params.ChainConfig) ([][]*types.Log, error) { // Retrieve the potentially incomplete receipts from disk or network - receipts := core.GetBlockReceipts(odr.Database(), hash, number) + receipts := core.GetBlockReceipts(odr.Database(), hash, number, config) if receipts == nil { r := &ReceiptsRequest{Hash: hash, Number: number} if err := odr.Retrieve(ctx, r); err != nil { diff --git a/light/txpool.go b/light/txpool.go index 0859b88dc0..420e84e8f3 100644 --- a/light/txpool.go +++ b/light/txpool.go @@ -183,7 +183,7 @@ func (pool *TxPool) checkMinedTxs(ctx context.Context, hash common.Hash, number // If some transactions have been mined, write the needed data to disk and update if list != nil { // Retrieve all the receipts belonging to this block and write the loopup table - if _, err := GetBlockReceipts(ctx, pool.odr, hash, number); err != nil { // ODR caches, ignore results + if _, err := GetBlockReceipts(ctx, pool.odr, hash, number, pool.config); err != nil { // ODR caches, ignore results return err } if err := core.WriteTxLookupEntries(pool.chainDb, block); err != nil { From 43a5a975f18003cb7a241573c08a5a4d2ea14a2c Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 5 Jul 2023 16:56:48 +0700 Subject: [PATCH 47/82] Optimize getStorageAt API --- internal/ethapi/api.go | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index e9ea6c4386..5ed4e61191 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -19,6 +19,7 @@ package ethapi import ( "bytes" "context" + "encoding/hex" "errors" "fmt" "math/big" @@ -518,6 +519,25 @@ type PublicBlockChainAPI struct { b Backend } +// decodeHash parses a hex-encoded 32-byte hash. The input may optionally +// be prefixed by 0x and can have a byte length up to 32. +func decodeHash(s string) (h common.Hash, inputLength int, err error) { + if strings.HasPrefix(s, "0x") || strings.HasPrefix(s, "0X") { + s = s[2:] + } + if (len(s) & 1) > 0 { + s = "0" + s + } + b, err := hex.DecodeString(s) + if err != nil { + return common.Hash{}, 0, errors.New("hex string invalid") + } + if len(b) > 32 { + return common.Hash{}, len(b), errors.New("hex string too long, want at most 32 bytes") + } + return common.BytesToHash(b), len(b), nil +} + // NewPublicBlockChainAPI creates a new Ethereum blockchain API. func NewPublicBlockChainAPI(b Backend) *PublicBlockChainAPI { return &PublicBlockChainAPI{b} @@ -639,12 +659,16 @@ func (s *PublicBlockChainAPI) GetCode(ctx context.Context, address common.Addres // GetStorageAt returns the storage from the state at the given address, key and // block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta block // numbers are also allowed. -func (s *PublicBlockChainAPI) GetStorageAt(ctx context.Context, address common.Address, key string, blockNr rpc.BlockNumber) (hexutil.Bytes, error) { +func (s *PublicBlockChainAPI) GetStorageAt(ctx context.Context, address common.Address, hexKey string, blockNr rpc.BlockNumber) (hexutil.Bytes, error) { state, _, err := s.b.StateAndHeaderByNumber(ctx, blockNr) if state == nil || err != nil { return nil, err } - res := state.GetState(address, common.HexToHash(key)) + key, _, err := decodeHash(hexKey) + if err != nil { + return nil, fmt.Errorf("unable to decode storage key: %s", err) + } + res := state.GetState(address, key) return res[:], state.Error() } From 78f392b2aa36d86ef16c4766c16291a7499f842d Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 5 Jul 2023 17:17:38 +0700 Subject: [PATCH 48/82] Fix NewTransactionsByPriceAndNonce --- core/types/transaction.go | 42 ++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/core/types/transaction.go b/core/types/transaction.go index 55d52f40b2..b3e5fb12ae 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -835,33 +835,25 @@ func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address]Transa delete(txs, from) continue } - //lastSpecialTx := -1 - //if len(signers) > 0 { - // if _, ok := signers[from]; ok { - // for i, tx := range accTxs { - // if tx.IsSpecialTransaction() { - // lastSpecialTx = i - // } - // } - // } - //} - //if lastSpecialTx >= 0 { - // for i := 0; i <= lastSpecialTx; i++ { - // specialTxs = append(specialTxs, accTxs[i]) - // } - //} else { - // heads.Push(wrapped) - // txs[from] = accTxs[1:] - //} - // TODO(trinhdn2): remove the for loop - for _, tx := range accTxs { - if tx.IsSpecialTransaction() { - specialTxs = append(specialTxs, tx) - } else { - heads.Push(wrapped) + // Exclude special transactions from accTxs of each address + lastSpecialTx := -1 + if len(signers) > 0 { + if _, ok := signers[from]; ok { + for i, tx := range accTxs { + if tx.IsSpecialTransaction() { + lastSpecialTx = i + } + } } - txs[from] = accTxs[1:] } + if lastSpecialTx >= 0 { + for i := 0; i <= lastSpecialTx; i++ { + specialTxs = append(specialTxs, accTxs[i]) + } + accTxs = accTxs[lastSpecialTx+1:] + } + heads.Push(wrapped) + txs[from] = accTxs[1:] } heap.Init(&heads) From 56fd5d4ed845ca68a59b1030107e76f649aa587e Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 5 Jul 2023 19:05:17 +0700 Subject: [PATCH 49/82] Force not ignoring special txs --- common/types.go | 17 +++++++++++++++-- core/state_transition.go | 2 +- core/types/transaction.go | 29 +++++++++++++++++------------ 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/common/types.go b/common/types.go index ca94230113..eb377f7779 100644 --- a/common/types.go +++ b/common/types.go @@ -51,8 +51,21 @@ const ( ) var ( - hashT = reflect.TypeOf(Hash{}) - addressT = reflect.TypeOf(Address{}) + hashT = reflect.TypeOf(Hash{}) + addressT = reflect.TypeOf(Address{}) + SpecialSMCAddressesMap = map[string]bool{ + BlockSigners: true, + MasternodeVotingSMC: true, + RandomizeSMC: true, + FoudationAddr: true, + TeamAddr: true, + TomoXAddr: true, + TradingStateAddr: true, + TomoXLendingAddress: true, + TomoXLendingFinalizedTradeAddress: true, + TomoNativeAddress: true, + LendingLockAddress: true, + } ) // Hash represents the 32 byte Keccak256 hash of arbitrary data. diff --git a/core/state_transition.go b/core/state_transition.go index 35c4e61c92..5ea25beea4 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -276,7 +276,7 @@ func (st *StateTransition) preCheck() error { } // This will panic if baseFee is nil, but BaseFee presence is verified // as part of header validation. - if msg.GasFeeCap.Cmp(st.evm.Context.BaseFee) < 0 { + if msg.GasFeeCap.Cmp(st.evm.Context.BaseFee) < 0 && !common.SpecialSMCAddressesMap[st.to().Address().String()] { return fmt.Errorf("%w: address %v, maxFeePerGas: %s baseFee: %s", ErrFeeCapTooLow, msg.From.Hex(), msg.GasFeeCap, st.evm.Context.BaseFee) } diff --git a/core/types/transaction.go b/core/types/transaction.go index b3e5fb12ae..bf37d7b66a 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -827,15 +827,9 @@ func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address]Transa payersSwap: payersSwap, } specialTxs := Transactions{} - for from, accTxs := range txs { - acc, _ := Sender(signer, accTxs[0]) - wrapped, err := NewTxWithMinerFee(accTxs[0], baseFee) - // Remove transaction if sender doesn't match from, or if wrapping fails. - if (acc != from || err != nil) && !accTxs[0].IsSpecialTransaction() { - delete(txs, from) - continue - } - // Exclude special transactions from accTxs of each address + for _, accTxs := range txs { + from, _ := Sender(signer, accTxs[0]) + var normalTxs Transactions lastSpecialTx := -1 if len(signers) > 0 { if _, ok := signers[from]; ok { @@ -850,10 +844,21 @@ func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address]Transa for i := 0; i <= lastSpecialTx; i++ { specialTxs = append(specialTxs, accTxs[i]) } - accTxs = accTxs[lastSpecialTx+1:] + normalTxs = accTxs[lastSpecialTx+1:] + } else { + normalTxs = accTxs + } + if len(normalTxs) > 0 { + wrapped, err := NewTxWithMinerFee(accTxs[0], baseFee) + // Remove transaction if sender doesn't match from, or if wrapping fails. + if err != nil { + delete(txs, from) + continue + } + heads.txs = append(heads.txs, wrapped) + // Ensure the sender address is from the signer + txs[from] = normalTxs[1:] } - heads.Push(wrapped) - txs[from] = accTxs[1:] } heap.Init(&heads) From 59ffc8291b99fe2d380ff7b9dbb9bab36c11f612 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 5 Jul 2023 19:24:25 +0700 Subject: [PATCH 50/82] Fix wrapping first normal tx in Fix NewTransactionsByPriceAndNonce --- core/types/transaction.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/types/transaction.go b/core/types/transaction.go index bf37d7b66a..4e0934c512 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -849,7 +849,7 @@ func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address]Transa normalTxs = accTxs } if len(normalTxs) > 0 { - wrapped, err := NewTxWithMinerFee(accTxs[0], baseFee) + wrapped, err := NewTxWithMinerFee(normalTxs[0], baseFee) // Remove transaction if sender doesn't match from, or if wrapping fails. if err != nil { delete(txs, from) From bcb96bf6016229cf3473b3085525169285eb2e7d Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 5 Jul 2023 20:12:35 +0700 Subject: [PATCH 51/82] Implement gas cap and eth_call timeout to prevent DoS --- cmd/utils/flags.go | 16 ++++++++++++++++ eth/api_backend.go | 14 ++++++++++++++ eth/config.go | 15 ++++++++++++++- eth/gen_config.go | 39 ++++++++++++++++++++++++++++++++++++++ internal/ethapi/api.go | 19 ++++++------------- internal/ethapi/backend.go | 4 ++++ les/api_backend.go | 13 +++++++++++++ 7 files changed, 106 insertions(+), 14 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 59a0cdeaf0..3acfbb5f72 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -367,6 +367,22 @@ var ( Name: "nocompaction", Usage: "Disables db compaction after import", } + // API options. + RPCGlobalGasCapFlag = &cli.Uint64Flag{ + Name: "rpc.gascap", + Usage: "Sets a cap on gas that can be used in eth_call/estimateGas (0=infinite)", + Value: eth.DefaultConfig.RPCGasCap, + } + RPCGlobalEVMTimeoutFlag = &cli.DurationFlag{ + Name: "rpc.evmtimeout", + Usage: "Sets a timeout used for eth_call (0=infinite)", + Value: eth.DefaultConfig.RPCEVMTimeout, + } + RPCGlobalTxFeeCapFlag = &cli.Float64Flag{ + Name: "rpc.txfeecap", + Usage: "Sets a cap on transaction fee (in ether) that can be sent via the RPC APIs (0 = no cap)", + Value: eth.DefaultConfig.RPCTxFeeCap, + } // RPC settings RPCEnabledFlag = cli.BoolFlag{ Name: "rpc", diff --git a/eth/api_backend.go b/eth/api_backend.go index 4164a83867..7204e1fc3e 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -24,6 +24,7 @@ import ( "io/ioutil" "math/big" "path/filepath" + "time" "github.com/tomochain/tomochain/tomox/tradingstate" "github.com/tomochain/tomochain/tomoxlending" @@ -264,6 +265,19 @@ func (b *EthApiBackend) SuggestGasTipCap(ctx context.Context) (*big.Int, error) return b.gpo.SuggestTipCap(ctx) } +func (b *EthApiBackend) RPCGasCap() uint64 { + return b.eth.config.RPCGasCap +} + +func (b *EthApiBackend) RPCEVMTimeout() time.Duration { + return b.eth.config.RPCEVMTimeout +} + +func (b *EthApiBackend) RPCTxFeeCap() float64 { + return b.eth.config.RPCTxFeeCap + +} + func (b *EthApiBackend) Downloader() *downloader.Downloader { return b.eth.Downloader() } diff --git a/eth/config.go b/eth/config.go index a86f084561..a68521e28d 100644 --- a/eth/config.go +++ b/eth/config.go @@ -55,6 +55,9 @@ var DefaultConfig = Config{ Blocks: 20, Percentile: 60, }, + RPCGasCap: 50000000, + RPCEVMTimeout: 5 * time.Second, + RPCTxFeeCap: 1, // 1 ether } func init() { @@ -71,7 +74,7 @@ func init() { } } -//go:generate gencodec -type Config -field-override configMarshaling -formats toml -out gen_config.go +//go:generate go run github.com/fjl/gencodec -type Config -field-override configMarshaling -formats toml -out gen_config.go type Config struct { // The genesis block, which is inserted if the database is empty. @@ -114,6 +117,16 @@ type Config struct { // Miscellaneous options DocRoot string `toml:"-"` + + // RPCGasCap is the global gas cap for eth-call variants. + RPCGasCap uint64 + + // RPCEVMTimeout is the global timeout for eth-call. + RPCEVMTimeout time.Duration + + // RPCTxFeeCap is the global transaction fee(price * gaslimit) cap for + // send-transaction variants. The unit is ether. + RPCTxFeeCap float64 } type configMarshaling struct { diff --git a/eth/gen_config.go b/eth/gen_config.go index 5464b7a792..514b70170e 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -4,6 +4,7 @@ package eth import ( "math/big" + "time" "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/common/hexutil" @@ -15,16 +16,20 @@ import ( var _ = (*configMarshaling)(nil) +// MarshalTOML marshals as TOML. func (c Config) MarshalTOML() (interface{}, error) { type Config struct { Genesis *core.Genesis `toml:",omitempty"` NetworkId uint64 SyncMode downloader.SyncMode + NoPruning bool LightServ int `toml:",omitempty"` LightPeers int `toml:",omitempty"` SkipBcVersionCheck bool `toml:"-"` DatabaseHandles int `toml:"-"` DatabaseCache int + TrieCache int + TrieTimeout time.Duration Etherbase common.Address `toml:",omitempty"` MinerThreads int `toml:",omitempty"` ExtraData hexutil.Bytes `toml:",omitempty"` @@ -34,16 +39,22 @@ func (c Config) MarshalTOML() (interface{}, error) { GPO gasprice.Config EnablePreimageRecording bool DocRoot string `toml:"-"` + RPCGasCap uint64 + RPCEVMTimeout time.Duration + RPCTxFeeCap float64 } var enc Config enc.Genesis = c.Genesis enc.NetworkId = c.NetworkId enc.SyncMode = c.SyncMode + enc.NoPruning = c.NoPruning enc.LightServ = c.LightServ enc.LightPeers = c.LightPeers enc.SkipBcVersionCheck = c.SkipBcVersionCheck enc.DatabaseHandles = c.DatabaseHandles enc.DatabaseCache = c.DatabaseCache + enc.TrieCache = c.TrieCache + enc.TrieTimeout = c.TrieTimeout enc.Etherbase = c.Etherbase enc.MinerThreads = c.MinerThreads enc.ExtraData = c.ExtraData @@ -53,19 +64,26 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.GPO = c.GPO enc.EnablePreimageRecording = c.EnablePreimageRecording enc.DocRoot = c.DocRoot + enc.RPCGasCap = c.RPCGasCap + enc.RPCEVMTimeout = c.RPCEVMTimeout + enc.RPCTxFeeCap = c.RPCTxFeeCap return &enc, nil } +// UnmarshalTOML unmarshals from TOML. func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { type Config struct { Genesis *core.Genesis `toml:",omitempty"` NetworkId *uint64 SyncMode *downloader.SyncMode + NoPruning *bool LightServ *int `toml:",omitempty"` LightPeers *int `toml:",omitempty"` SkipBcVersionCheck *bool `toml:"-"` DatabaseHandles *int `toml:"-"` DatabaseCache *int + TrieCache *int + TrieTimeout *time.Duration Etherbase *common.Address `toml:",omitempty"` MinerThreads *int `toml:",omitempty"` ExtraData *hexutil.Bytes `toml:",omitempty"` @@ -75,6 +93,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { GPO *gasprice.Config EnablePreimageRecording *bool DocRoot *string `toml:"-"` + RPCGasCap *uint64 + RPCEVMTimeout *time.Duration + RPCTxFeeCap *float64 } var dec Config if err := unmarshal(&dec); err != nil { @@ -89,6 +110,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.SyncMode != nil { c.SyncMode = *dec.SyncMode } + if dec.NoPruning != nil { + c.NoPruning = *dec.NoPruning + } if dec.LightServ != nil { c.LightServ = *dec.LightServ } @@ -104,6 +128,12 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.DatabaseCache != nil { c.DatabaseCache = *dec.DatabaseCache } + if dec.TrieCache != nil { + c.TrieCache = *dec.TrieCache + } + if dec.TrieTimeout != nil { + c.TrieTimeout = *dec.TrieTimeout + } if dec.Etherbase != nil { c.Etherbase = *dec.Etherbase } @@ -131,5 +161,14 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.DocRoot != nil { c.DocRoot = *dec.DocRoot } + if dec.RPCGasCap != nil { + c.RPCGasCap = *dec.RPCGasCap + } + if dec.RPCEVMTimeout != nil { + c.RPCEVMTimeout = *dec.RPCEVMTimeout + } + if dec.RPCTxFeeCap != nil { + c.RPCTxFeeCap = *dec.RPCTxFeeCap + } return nil } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 5ed4e61191..439bd192df 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1059,22 +1059,12 @@ func (s *PublicBlockChainAPI) getCandidatesFromSmartContract() ([]posv.Masternod return candidatesWithStakeInfo, nil } -func (s *PublicBlockChainAPI) doCall(ctx context.Context, args TransactionArgs, blockNr rpc.BlockNumber, timeout time.Duration) ([]byte, uint64, bool, error) { +func (s *PublicBlockChainAPI) doCall(ctx context.Context, args TransactionArgs, blockNr rpc.BlockNumber, timeout time.Duration, globalGasCap uint64) ([]byte, uint64, bool, error) { defer func(start time.Time) { log.Debug("Executing EVM call finished", "runtime", time.Since(start)) }(time.Now()) - statedb, header, err := s.b.StateAndHeaderByNumber(ctx, blockNr) if statedb == nil || err != nil { return nil, 0, false, err } - // Set sender address or use a default if none specified - addr := args.From - if *addr == (common.Address{}) { - if wallets := s.b.AccountManager().Wallets(); len(wallets) > 0 { - if accounts := wallets[0].Accounts(); len(accounts) > 0 { - addr = &accounts[0].Address - } - } - } // Set default gas price if none were set. gasPrice := args.GasPrice.ToInt() if gasPrice == nil { @@ -1083,6 +1073,9 @@ func (s *PublicBlockChainAPI) doCall(ctx context.Context, args TransactionArgs, if gasPrice == nil || gasPrice.Sign() == 0 { gasPrice = new(big.Int).SetUint64(defaultGasPrice) } + if args.Gas == nil { + args.Gas = (*hexutil.Uint64)(&globalGasCap) + } balanceTokenFee := big.NewInt(0).SetUint64(uint64(*args.Gas)) balanceTokenFee = balanceTokenFee.Mul(balanceTokenFee, gasPrice) // Create new call message @@ -1142,7 +1135,7 @@ func (s *PublicBlockChainAPI) doCall(ctx context.Context, args TransactionArgs, // Call executes the given transaction on the state for the given block number. // It doesn't make and changes in the state/blockchain and is useful to execute and retrieve values. func (s *PublicBlockChainAPI) Call(ctx context.Context, args TransactionArgs, blockNr rpc.BlockNumber) (hexutil.Bytes, error) { - result, _, _, err := s.doCall(ctx, args, blockNr, 5*time.Second) + result, _, _, err := s.doCall(ctx, args, blockNr, s.b.RPCEVMTimeout(), s.b.RPCGasCap()) return (hexutil.Bytes)(result), err } @@ -1218,7 +1211,7 @@ func (s *PublicBlockChainAPI) EstimateGas(ctx context.Context, args TransactionA executable := func(gas uint64) bool { args.Gas = (*hexutil.Uint64)(&gas) - _, _, failed, err := s.doCall(ctx, args, rpc.LatestBlockNumber, 0) + _, _, failed, err := s.doCall(ctx, args, rpc.LatestBlockNumber, 0, s.b.RPCGasCap()) if err != nil || failed { return false } diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index 5500e9d1fb..afc3a7f2db 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -20,6 +20,7 @@ package ethapi import ( "context" "math/big" + "time" "github.com/tomochain/tomochain/accounts" "github.com/tomochain/tomochain/common" @@ -50,6 +51,9 @@ type Backend interface { AccountManager() *accounts.Manager TomoxService() *tomox.TomoX LendingService() *tomoxlending.Lending + RPCGasCap() uint64 // global gas cap for eth_call over rpc: DoS protection + RPCEVMTimeout() time.Duration // global timeout for eth_call over rpc: DoS protection + RPCTxFeeCap() float64 // global tx fee cap for all transaction related APIs SuggestGasTipCap(ctx context.Context) (*big.Int, error) // BlockChain API diff --git a/les/api_backend.go b/les/api_backend.go index 3e5b035802..e82efa5447 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -23,6 +23,7 @@ import ( "io/ioutil" "math/big" "path/filepath" + "time" "github.com/tomochain/tomochain/accounts" "github.com/tomochain/tomochain/common" @@ -220,6 +221,18 @@ func (b *LesApiBackend) SuggestGasTipCap(ctx context.Context) (*big.Int, error) return b.gpo.SuggestTipCap(ctx) } +func (b *LesApiBackend) RPCGasCap() uint64 { + return b.eth.config.RPCGasCap +} + +func (b *LesApiBackend) RPCEVMTimeout() time.Duration { + return b.eth.config.RPCEVMTimeout +} + +func (b *LesApiBackend) RPCTxFeeCap() float64 { + return b.eth.config.RPCTxFeeCap +} + func (b *LesApiBackend) Downloader() *downloader.Downloader { return b.eth.Downloader() } From a399863607ebaf9e4e24704716dfa58703dc8262 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 5 Jul 2023 20:17:41 +0700 Subject: [PATCH 52/82] Add more RPC flags --- cmd/tomo/main.go | 3 +++ cmd/utils/flags.go | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/cmd/tomo/main.go b/cmd/tomo/main.go index 2a606fbb78..b39eafa95e 100644 --- a/cmd/tomo/main.go +++ b/cmd/tomo/main.go @@ -143,6 +143,9 @@ var ( utils.WSAllowedOriginsFlag, utils.IPCDisabledFlag, utils.IPCPathFlag, + utils.RPCGlobalGasCapFlag, + utils.RPCGlobalEVMTimeoutFlag, + utils.RPCGlobalTxFeeCapFlag, } whisperFlags = []cli.Flag{ diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 3acfbb5f72..09e46bb28f 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1147,6 +1147,20 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(DocRootFlag.Name) { cfg.DocRoot = ctx.GlobalString(DocRootFlag.Name) } + if ctx.IsSet(RPCGlobalGasCapFlag.Name) { + cfg.RPCGasCap = ctx.Uint64(RPCGlobalGasCapFlag.Name) + } + if cfg.RPCGasCap != 0 { + log.Info("Set global gas cap", "cap", cfg.RPCGasCap) + } else { + log.Info("Global gas cap disabled") + } + if ctx.IsSet(RPCGlobalEVMTimeoutFlag.Name) { + cfg.RPCEVMTimeout = ctx.Duration(RPCGlobalEVMTimeoutFlag.Name) + } + if ctx.IsSet(RPCGlobalTxFeeCapFlag.Name) { + cfg.RPCTxFeeCap = ctx.Float64(RPCGlobalTxFeeCapFlag.Name) + } if ctx.GlobalIsSet(ExtraDataFlag.Name) { cfg.ExtraData = []byte(ctx.GlobalString(ExtraDataFlag.Name)) } From b8de2d70a7cda2d2db0bbe31fe85a12a708a10b7 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Thu, 6 Jul 2023 00:25:04 +0700 Subject: [PATCH 53/82] Fix receipt RLP panic --- core/types/gen_receipt_json.go | 32 ++++++++++++++++---------------- core/types/log.go | 3 ++- core/types/receipt.go | 11 +++++++---- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/core/types/gen_receipt_json.go b/core/types/gen_receipt_json.go index 06f61c2e20..5075b2921d 100644 --- a/core/types/gen_receipt_json.go +++ b/core/types/gen_receipt_json.go @@ -16,7 +16,7 @@ var _ = (*receiptMarshaling)(nil) // MarshalJSON marshals as JSON. func (r Receipt) MarshalJSON() ([]byte, error) { type Receipt struct { - Type uint8 `json:"type,omitempty"` + Type hexutil.Uint64 `json:"type,omitempty"` PostState hexutil.Bytes `json:"root"` Status hexutil.Uint64 `json:"status"` CumulativeGasUsed hexutil.Uint64 `json:"cumulativeGasUsed" gencodec:"required"` @@ -25,13 +25,13 @@ func (r Receipt) MarshalJSON() ([]byte, error) { TxHash common.Hash `json:"transactionHash" gencodec:"required"` ContractAddress common.Address `json:"contractAddress"` GasUsed hexutil.Uint64 `json:"gasUsed" gencodec:"required"` - EffectiveGasPrice *big.Int `json:"effectiveGasPrice"` + EffectiveGasPrice *hexutil.Big `json:"effectiveGasPrice"` BlockHash common.Hash `json:"blockHash,omitempty"` - BlockNumber *big.Int `json:"blockNumber,omitempty"` - TransactionIndex uint `json:"transactionIndex"` + BlockNumber *hexutil.Big `json:"blockNumber,omitempty"` + TransactionIndex hexutil.Uint `json:"transactionIndex"` } var enc Receipt - enc.Type = r.Type + enc.Type = hexutil.Uint64(r.Type) enc.PostState = r.PostState enc.Status = hexutil.Uint64(r.Status) enc.CumulativeGasUsed = hexutil.Uint64(r.CumulativeGasUsed) @@ -40,17 +40,17 @@ func (r Receipt) MarshalJSON() ([]byte, error) { enc.TxHash = r.TxHash enc.ContractAddress = r.ContractAddress enc.GasUsed = hexutil.Uint64(r.GasUsed) - enc.EffectiveGasPrice = r.EffectiveGasPrice + enc.EffectiveGasPrice = (*hexutil.Big)(r.EffectiveGasPrice) enc.BlockHash = r.BlockHash - enc.BlockNumber = r.BlockNumber - enc.TransactionIndex = r.TransactionIndex + enc.BlockNumber = (*hexutil.Big)(r.BlockNumber) + enc.TransactionIndex = hexutil.Uint(r.TransactionIndex) return json.Marshal(&enc) } // UnmarshalJSON unmarshals from JSON. func (r *Receipt) UnmarshalJSON(input []byte) error { type Receipt struct { - Type *uint8 `json:"type,omitempty"` + Type *hexutil.Uint64 `json:"type,omitempty"` PostState *hexutil.Bytes `json:"root"` Status *hexutil.Uint64 `json:"status"` CumulativeGasUsed *hexutil.Uint64 `json:"cumulativeGasUsed" gencodec:"required"` @@ -59,17 +59,17 @@ func (r *Receipt) UnmarshalJSON(input []byte) error { TxHash *common.Hash `json:"transactionHash" gencodec:"required"` ContractAddress *common.Address `json:"contractAddress"` GasUsed *hexutil.Uint64 `json:"gasUsed" gencodec:"required"` - EffectiveGasPrice *big.Int `json:"effectiveGasPrice"` + EffectiveGasPrice *hexutil.Big `json:"effectiveGasPrice"` BlockHash *common.Hash `json:"blockHash,omitempty"` - BlockNumber *big.Int `json:"blockNumber,omitempty"` - TransactionIndex *uint `json:"transactionIndex"` + BlockNumber *hexutil.Big `json:"blockNumber,omitempty"` + TransactionIndex *hexutil.Uint `json:"transactionIndex"` } var dec Receipt if err := json.Unmarshal(input, &dec); err != nil { return err } if dec.Type != nil { - r.Type = *dec.Type + r.Type = uint8(*dec.Type) } if dec.PostState != nil { r.PostState = *dec.PostState @@ -101,16 +101,16 @@ func (r *Receipt) UnmarshalJSON(input []byte) error { } r.GasUsed = uint64(*dec.GasUsed) if dec.EffectiveGasPrice != nil { - r.EffectiveGasPrice = dec.EffectiveGasPrice + r.EffectiveGasPrice = (*big.Int)(dec.EffectiveGasPrice) } if dec.BlockHash != nil { r.BlockHash = *dec.BlockHash } if dec.BlockNumber != nil { - r.BlockNumber = dec.BlockNumber + r.BlockNumber = (*big.Int)(dec.BlockNumber) } if dec.TransactionIndex != nil { - r.TransactionIndex = *dec.TransactionIndex + r.TransactionIndex = uint(*dec.TransactionIndex) } return nil } diff --git a/core/types/log.go b/core/types/log.go index 93567b1e61..77e79f24a3 100644 --- a/core/types/log.go +++ b/core/types/log.go @@ -85,7 +85,8 @@ type rlpStorageLog struct { // EncodeRLP implements rlp.Encoder. func (l *Log) EncodeRLP(w io.Writer) error { - return rlp.Encode(w, rlpLog{Address: l.Address, Topics: l.Topics, Data: l.Data}) + rl := rlpLog{Address: l.Address, Topics: l.Topics, Data: l.Data} + return rlp.Encode(w, &rl) } // DecodeRLP implements rlp.Decoder. diff --git a/core/types/receipt.go b/core/types/receipt.go index d8a197089a..45e6feeb18 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -24,11 +24,10 @@ import ( "math/big" "unsafe" - "github.com/tomochain/tomochain/crypto" - "github.com/tomochain/tomochain/params" - "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/common/hexutil" + "github.com/tomochain/tomochain/crypto" + "github.com/tomochain/tomochain/params" "github.com/tomochain/tomochain/rlp" ) @@ -73,10 +72,14 @@ type Receipt struct { } type receiptMarshaling struct { + Type hexutil.Uint64 PostState hexutil.Bytes Status hexutil.Uint64 CumulativeGasUsed hexutil.Uint64 GasUsed hexutil.Uint64 + EffectiveGasPrice *hexutil.Big + BlockNumber *hexutil.Big + TransactionIndex hexutil.Uint } // receiptRLP is the consensus encoding of a receipt. @@ -361,7 +364,7 @@ func (rs Receipts) DeriveFields(config *params.ChainConfig, hash common.Hash, nu return nil } -// GetRlp returns the RLP encoding of one receipt from the list. +// GetRlp returns the RLP encoding of one receipt from the list.. func (r Receipts) GetRlp(i int) []byte { bytes, err := rlp.EncodeToBytes(r[i]) if err != nil { From 938ab4b9b82fd9627e5b964493d41b55ff55e29a Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Thu, 6 Jul 2023 14:15:50 +0700 Subject: [PATCH 54/82] Minor fix --- cmd/puppeth/wizard_genesis.go | 4 ++++ core/types/transaction.go | 3 +++ internal/ethapi/api.go | 2 -- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/cmd/puppeth/wizard_genesis.go b/cmd/puppeth/wizard_genesis.go index a2ceb56774..f8f423c11b 100644 --- a/cmd/puppeth/wizard_genesis.go +++ b/cmd/puppeth/wizard_genesis.go @@ -387,6 +387,10 @@ func (w *wizard) manageGenesis() { fmt.Printf("Which block should Byzantium come into effect? (default = %v)\n", w.conf.Genesis.Config.ByzantiumBlock) w.conf.Genesis.Config.ByzantiumBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ByzantiumBlock) + fmt.Println() + fmt.Printf("Which block should London come into effect? (default = %v)\n", w.conf.Genesis.Config.LondonBlock) + w.conf.Genesis.Config.LondonBlock = w.readDefaultBigInt(w.conf.Genesis.Config.LondonBlock) + out, _ := json.MarshalIndent(w.conf.Genesis.Config, "", " ") fmt.Printf("Chain configuration updated:\n\n%s\n", out) diff --git a/core/types/transaction.go b/core/types/transaction.go index 4e0934c512..c682596c01 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -828,6 +828,9 @@ func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address]Transa } specialTxs := Transactions{} for _, accTxs := range txs { + if len(accTxs) == 0 { + continue + } from, _ := Sender(signer, accTxs[0]) var normalTxs Transactions lastSpecialTx := -1 diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 439bd192df..59295255a6 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1079,8 +1079,6 @@ func (s *PublicBlockChainAPI) doCall(ctx context.Context, args TransactionArgs, balanceTokenFee := big.NewInt(0).SetUint64(uint64(*args.Gas)) balanceTokenFee = balanceTokenFee.Mul(balanceTokenFee, gasPrice) // Create new call message - - // Get a new instance of the EVM. msg, err := args.ToMessage(header.BaseFee, balanceTokenFee) if err != nil { return nil, 0, false, err From a8e89d9b5f1d04ac7e81ad2b33e594456a09afa0 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Thu, 6 Jul 2023 23:30:56 +0700 Subject: [PATCH 55/82] Revert GetBlockReceipts interface and some minor fixes --- accounts/abi/bind/backends/simulated.go | 4 +-- cmd/tomo/bugcmd.go | 3 +- console/console_test.go | 25 +++++-------- core/bench_test.go | 2 +- core/blockchain_test.go | 10 +++--- core/database_util_test.go | 9 ++--- core/genesis.go | 19 +++++++--- core/types/transaction.go | 3 +- eth/api_backend.go | 4 +-- eth/config.go | 8 +++-- eth/filters/filter.go | 4 +-- eth/filters/filter_system.go | 2 +- eth/filters/filter_system_test.go | 10 ++++-- internal/ethapi/api.go | 2 +- internal/ethapi/backend.go | 2 +- les/api_backend.go | 4 +-- les/handler_test.go | 7 ++-- les/odr_test.go | 48 +++++++++++++------------ params/config.go | 2 +- tests/block_test_util.go | 19 +++++----- 20 files changed, 102 insertions(+), 85 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 1b55704812..7bebd34984 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -569,8 +569,8 @@ func (fb *filterBackend) HeaderByNumber(ctx context.Context, block rpc.BlockNumb return fb.bc.GetHeaderByNumber(uint64(block.Int64())), nil } -func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash, config *params.ChainConfig) (types.Receipts, error) { - return core.GetBlockReceipts(fb.db, hash, core.GetBlockNumber(fb.db, hash), config), nil +func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { + return core.GetBlockReceipts(fb.db, hash, core.GetBlockNumber(fb.db, hash), fb.ChainConfig()), nil } func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { diff --git a/cmd/tomo/bugcmd.go b/cmd/tomo/bugcmd.go index 3174f73881..5cec10ad49 100644 --- a/cmd/tomo/bugcmd.go +++ b/cmd/tomo/bugcmd.go @@ -105,5 +105,4 @@ const header = `Please answer these questions before submitting your issue. Than #### What did you see instead? -#### System details -` +#### System details` diff --git a/console/console_test.go b/console/console_test.go index 22527f4ddc..9565c7e97a 100644 --- a/console/console_test.go +++ b/console/console_test.go @@ -19,20 +19,19 @@ package console import ( "bytes" "errors" - "github.com/tomochain/tomochain/tomox" - "github.com/tomochain/tomochain/tomoxlending" - "io/ioutil" "os" "strings" "testing" "time" "github.com/tomochain/tomochain/common" - "github.com/tomochain/tomochain/consensus/ethash" + "github.com/tomochain/tomochain/console/prompt" "github.com/tomochain/tomochain/core" "github.com/tomochain/tomochain/eth" "github.com/tomochain/tomochain/internal/jsre" "github.com/tomochain/tomochain/node" + "github.com/tomochain/tomochain/tomox" + "github.com/tomochain/tomochain/tomoxlending" ) const ( @@ -67,10 +66,10 @@ func (p *hookedPrompter) PromptPassword(prompt string) (string, error) { func (p *hookedPrompter) PromptConfirm(prompt string) (bool, error) { return false, errors.New("not implemented") } -func (p *hookedPrompter) SetHistory(history []string) {} -func (p *hookedPrompter) AppendHistory(command string) {} -func (p *hookedPrompter) ClearHistory() {} -func (p *hookedPrompter) SetWordCompleter(completer WordCompleter) {} +func (p *hookedPrompter) SetHistory(history []string) {} +func (p *hookedPrompter) AppendHistory(command string) {} +func (p *hookedPrompter) ClearHistory() {} +func (p *hookedPrompter) SetWordCompleter(completer prompt.WordCompleter) {} // tester is a console test environment for the console tests to operate on. type tester struct { @@ -86,10 +85,7 @@ type tester struct { // Please ensure you call Close() on the returned tester to avoid leaks. func newTester(t *testing.T, confOverride func(*eth.Config)) *tester { // Create a temporary storage for the node keys and initialize it - workspace, err := ioutil.TempDir("", "console-tester-") - if err != nil { - t.Fatalf("failed to create temporary keystore: %v", err) - } + workspace := t.TempDir() // Create a networkless protocol stack and start an Ethereum service within stack, err := node.New(&node.Config{DataDir: workspace, UseLightweightKDF: true, Name: testInstance}) @@ -99,9 +95,6 @@ func newTester(t *testing.T, confOverride func(*eth.Config)) *tester { ethConf := ð.Config{ Genesis: core.DeveloperGenesisBlock(15, common.Address{}), Etherbase: common.HexToAddress(testAddress), - Ethash: ethash.Config{ - PowMode: ethash.ModeTest, - }, } if confOverride != nil { confOverride(ethConf) @@ -262,7 +255,7 @@ func TestPrettyError(t *testing.T) { defer tester.Close(t) tester.console.Evaluate("throw 'hello'") - want := jsre.ErrorColor("hello") + "\n" + want := jsre.ErrorColor("hello") + "\n\tat :1:1(1)\n\n" if output := tester.output.String(); output != want { t.Fatalf("pretty error mismatch: have %s, want %s", output, want) } diff --git a/core/bench_test.go b/core/bench_test.go index a52aa7f2c2..d0a90b1ec3 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -294,7 +294,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) { if full { hash := header.Hash() GetBody(db, hash, n) - GetBlockReceipts(db, hash, n) + GetBlockReceipts(db, hash, n, params.TestChainConfig) } } diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 6860924112..cf209d9f99 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -18,7 +18,6 @@ package core import ( "fmt" - "github.com/tomochain/tomochain/core/rawdb" "math/big" "math/rand" "sync" @@ -27,6 +26,7 @@ import ( "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/consensus/ethash" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/state" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/core/vm" @@ -622,7 +622,7 @@ func TestFastVsFullChains(t *testing.T) { } else if types.CalcUncleHash(fblock.Uncles()) != types.CalcUncleHash(ablock.Uncles()) { t.Errorf("block #%d [%x]: uncles mismatch: have %v, want %v", num, hash, fblock.Uncles(), ablock.Uncles()) } - if freceipts, areceipts := GetBlockReceipts(fastDb, hash, GetBlockNumber(fastDb, hash)), GetBlockReceipts(archiveDb, hash, GetBlockNumber(archiveDb, hash)); types.DeriveSha(freceipts) != types.DeriveSha(areceipts) { + if freceipts, areceipts := GetBlockReceipts(fastDb, hash, GetBlockNumber(fastDb, hash), fast.Config()), GetBlockReceipts(archiveDb, hash, GetBlockNumber(archiveDb, hash), fast.Config()); types.DeriveSha(freceipts) != types.DeriveSha(areceipts) { t.Errorf("block #%d [%x]: receipts mismatch: have %v, want %v", num, hash, freceipts, areceipts) } } @@ -807,7 +807,7 @@ func TestChainTxReorgs(t *testing.T) { if txn, _, _, _ := GetTransaction(db, tx.Hash()); txn != nil { t.Errorf("drop %d: tx %v found while shouldn't have been", i, txn) } - if rcpt, _, _, _ := GetReceipt(db, tx.Hash()); rcpt != nil { + if rcpt, _, _, _ := GetReceipt(db, tx.Hash(), blockchain.Config()); rcpt != nil { t.Errorf("drop %d: receipt %v found while shouldn't have been", i, rcpt) } } @@ -816,7 +816,7 @@ func TestChainTxReorgs(t *testing.T) { if txn, _, _, _ := GetTransaction(db, tx.Hash()); txn == nil { t.Errorf("add %d: expected tx to be found", i) } - if rcpt, _, _, _ := GetReceipt(db, tx.Hash()); rcpt == nil { + if rcpt, _, _, _ := GetReceipt(db, tx.Hash(), blockchain.Config()); rcpt == nil { t.Errorf("add %d: expected receipt to be found", i) } } @@ -825,7 +825,7 @@ func TestChainTxReorgs(t *testing.T) { if txn, _, _, _ := GetTransaction(db, tx.Hash()); txn == nil { t.Errorf("share %d: expected tx to be found", i) } - if rcpt, _, _, _ := GetReceipt(db, tx.Hash()); rcpt == nil { + if rcpt, _, _, _ := GetReceipt(db, tx.Hash(), blockchain.Config()); rcpt == nil { t.Errorf("share %d: expected receipt to be found", i) } } diff --git a/core/database_util_test.go b/core/database_util_test.go index f28ca160a5..ab57e8876e 100644 --- a/core/database_util_test.go +++ b/core/database_util_test.go @@ -18,13 +18,14 @@ package core import ( "bytes" - "github.com/tomochain/tomochain/core/rawdb" "math/big" "testing" "github.com/tomochain/tomochain/common" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/crypto/sha3" + "github.com/tomochain/tomochain/params" "github.com/tomochain/tomochain/rlp" ) @@ -361,14 +362,14 @@ func TestBlockReceiptStorage(t *testing.T) { // Check that no receipt entries are in a pristine database hash := common.BytesToHash([]byte{0x03, 0x14}) - if rs := GetBlockReceipts(db, hash, 0); len(rs) != 0 { + if rs := GetBlockReceipts(db, hash, 0, params.TestChainConfig); len(rs) != 0 { t.Fatalf("non existent receipts returned: %v", rs) } // Insert the receipt slice into the database and check presence if err := WriteBlockReceipts(db, hash, 0, receipts); err != nil { t.Fatalf("failed to write block receipts: %v", err) } - if rs := GetBlockReceipts(db, hash, 0); len(rs) == 0 { + if rs := GetBlockReceipts(db, hash, 0, params.TestChainConfig); len(rs) == 0 { t.Fatalf("no receipts returned") } else { for i := 0; i < len(receipts); i++ { @@ -382,7 +383,7 @@ func TestBlockReceiptStorage(t *testing.T) { } // Delete the receipt slice and check purge DeleteBlockReceipts(db, hash, 0) - if rs := GetBlockReceipts(db, hash, 0); len(rs) != 0 { + if rs := GetBlockReceipts(db, hash, 0, params.TestChainConfig); len(rs) != 0 { t.Fatalf("deleted receipts returned: %v", rs) } } diff --git a/core/genesis.go b/core/genesis.go index e1b7185a41..8b8d0dd158 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -22,13 +22,13 @@ import ( "encoding/json" "errors" "fmt" - "github.com/tomochain/tomochain/core/rawdb" "math/big" "strings" "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/common/hexutil" "github.com/tomochain/tomochain/common/math" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/state" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/ethdb" @@ -60,6 +60,7 @@ type Genesis struct { Number uint64 `json:"number"` GasUsed uint64 `json:"gasUsed"` ParentHash common.Hash `json:"parentHash"` + BaseFee *big.Int `json:"baseFeePerGas"` } // GenesisAlloc specifies the initial state that is part of the genesis block. @@ -140,10 +141,10 @@ func (e *GenesisMismatchError) Error() string { // SetupGenesisBlock writes or updates the genesis block in db. // The block that will be used is: // -// genesis == nil genesis != nil -// +------------------------------------------ -// db has no genesis | main-net default | genesis -// db has genesis | from DB | genesis (if compatible) +// genesis == nil genesis != nil +// +------------------------------------------ +// db has no genesis | main-net default | genesis +// db has genesis | from DB | genesis (if compatible) // // The stored chain configuration will be updated if it is compatible (i.e. does not // specify a fork block below the local head block). In case of a conflict, the @@ -255,6 +256,13 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { if g.Difficulty == nil { head.Difficulty = params.GenesisDifficulty } + if g.Config != nil && g.Config.IsLondon(common.Big0) { + if g.BaseFee != nil { + head.BaseFee = g.BaseFee + } else { + head.BaseFee = new(big.Int).SetUint64(params.InitialBaseFee) + } + } statedb.Commit(false) statedb.Database().TrieDB().Commit(root, true) @@ -358,6 +366,7 @@ func DeveloperGenesisBlock(period uint64, faucet common.Address) *Genesis { Config: &config, ExtraData: append(append(make([]byte, 32), faucet[:]...), make([]byte, 65)...), GasLimit: 6283185, + BaseFee: big.NewInt(params.InitialBaseFee), Difficulty: big.NewInt(1), Alloc: map[common.Address]GenesisAccount{ common.BytesToAddress([]byte{1}): {Balance: big.NewInt(1)}, // ECRecover diff --git a/core/types/transaction.go b/core/types/transaction.go index c682596c01..5f4851d375 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -828,7 +828,8 @@ func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address]Transa } specialTxs := Transactions{} for _, accTxs := range txs { - if len(accTxs) == 0 { + // check because sometimes the map value can be null + if accTxs == nil || len(accTxs) == 0 { continue } from, _ := Sender(signer, accTxs[0]) diff --git a/eth/api_backend.go b/eth/api_backend.go index 7204e1fc3e..ed3768dd27 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -160,8 +160,8 @@ func (b *EthApiBackend) GetBlock(ctx context.Context, blockHash common.Hash) (*t return b.eth.blockchain.GetBlockByHash(blockHash), nil } -func (b *EthApiBackend) GetReceipts(ctx context.Context, blockHash common.Hash, config *params.ChainConfig) (types.Receipts, error) { - return core.GetBlockReceipts(b.eth.chainDb, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash), config), nil +func (b *EthApiBackend) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) { + return core.GetBlockReceipts(b.eth.chainDb, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash), b.ChainConfig()), nil } func (b *EthApiBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) { diff --git a/eth/config.go b/eth/config.go index a68521e28d..7390feec47 100644 --- a/eth/config.go +++ b/eth/config.go @@ -52,8 +52,12 @@ var DefaultConfig = Config{ TxPool: core.DefaultTxPoolConfig, GPO: gasprice.Config{ - Blocks: 20, - Percentile: 60, + Blocks: 20, + Percentile: 60, + MaxHeaderHistory: 1024, + MaxBlockHistory: 1024, + MaxPrice: gasprice.DefaultMaxPrice, + IgnorePrice: gasprice.DefaultIgnorePrice, }, RPCGasCap: 50000000, RPCEVMTimeout: 5 * time.Second, diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 7f0ea75fe1..7767deb0ed 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -34,7 +34,7 @@ type Backend interface { ChainDb() ethdb.Database EventMux() *event.TypeMux HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) - GetReceipts(ctx context.Context, blockHash common.Hash, config *params.ChainConfig) (types.Receipts, error) + GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) ChainConfig() *params.ChainConfig @@ -216,7 +216,7 @@ func (f *Filter) checkMatches(ctx context.Context, header *types.Header) (logs [ if len(logs) > 0 { // We have matching logs, check if we need to resolve full logs via the light client if logs[0].TxHash == (common.Hash{}) { - receipts, err := f.backend.GetReceipts(ctx, header.Hash(), f.backend.ChainConfig()) + receipts, err := f.backend.GetReceipts(ctx, header.Hash()) if err != nil { return nil, err } diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index 4c15d29a88..3d92fc1ac7 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -390,7 +390,7 @@ func (es *EventSystem) lightFilterLogs(header *types.Header, addresses []common. logs := filterLogs(unfiltered, nil, nil, addresses, topics) if len(logs) > 0 && logs[0].TxHash == (common.Hash{}) { // We have matching but non-derived logs - receipts, err := es.backend.GetReceipts(ctx, header.Hash(), es.backend.ChainConfig()) + receipts, err := es.backend.GetReceipts(ctx, header.Hash()) if err != nil { return nil } diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index d947a672ac..eb3e7cce4b 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -19,7 +19,6 @@ package filters import ( "context" "fmt" - "github.com/tomochain/tomochain/core/rawdb" "math/big" "math/rand" "reflect" @@ -31,6 +30,7 @@ import ( "github.com/tomochain/tomochain/consensus/ethash" "github.com/tomochain/tomochain/core" "github.com/tomochain/tomochain/core/bloombits" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/ethdb" "github.com/tomochain/tomochain/event" @@ -48,6 +48,10 @@ type testBackend struct { chainFeed *event.Feed } +func (b *testBackend) ChainConfig() *params.ChainConfig { + return params.TestChainConfig +} + func (b *testBackend) ChainDb() ethdb.Database { return b.db } @@ -71,12 +75,12 @@ func (b *testBackend) HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumbe func (b *testBackend) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) { number := core.GetBlockNumber(b.db, blockHash) - return core.GetBlockReceipts(b.db, blockHash, number), nil + return core.GetBlockReceipts(b.db, blockHash, number, b.ChainConfig()), nil } func (b *testBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) { number := core.GetBlockNumber(b.db, blockHash) - receipts := core.GetBlockReceipts(b.db, blockHash, number) + receipts := core.GetBlockReceipts(b.db, blockHash, number, b.ChainConfig()) logs := make([][]*types.Log, len(receipts)) for i, receipt := range receipts { diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 59295255a6..0b7e8e6979 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1849,7 +1849,7 @@ func (s *PublicTransactionPoolAPI) GetTransactionReceipt(ctx context.Context, ha if err != nil { return nil, err } - receipts, err := s.b.GetReceipts(ctx, blockHash, s.b.ChainConfig()) + receipts, err := s.b.GetReceipts(ctx, blockHash) if err != nil { return nil, err } diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index afc3a7f2db..9a88c2310b 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -66,7 +66,7 @@ type Backend interface { StateAndHeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*state.StateDB, *types.Header, error) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) GetBlock(ctx context.Context, blockHash common.Hash) (*types.Block, error) - GetReceipts(ctx context.Context, blockHash common.Hash, config *params.ChainConfig) (types.Receipts, error) + GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) GetTd(blockHash common.Hash) *big.Int GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, tomoxState *tradingstate.TradingStateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription diff --git a/les/api_backend.go b/les/api_backend.go index e82efa5447..15a3ebe8e2 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -134,8 +134,8 @@ func (b *LesApiBackend) GetBlock(ctx context.Context, blockHash common.Hash) (*t return b.eth.blockchain.GetBlockByHash(ctx, blockHash) } -func (b *LesApiBackend) GetReceipts(ctx context.Context, blockHash common.Hash, config *params.ChainConfig) (types.Receipts, error) { - return light.GetBlockReceipts(ctx, b.eth.odr, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash), config) +func (b *LesApiBackend) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) { + return light.GetBlockReceipts(ctx, b.eth.odr, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash), b.ChainConfig()) } func (b *LesApiBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) { diff --git a/les/handler_test.go b/les/handler_test.go index 225900dd52..a5c839cb0a 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -18,7 +18,6 @@ package les import ( "encoding/binary" - "github.com/tomochain/tomochain/core/rawdb" "math/big" "math/rand" "testing" @@ -27,6 +26,7 @@ import ( "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/consensus/ethash" "github.com/tomochain/tomochain/core" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/crypto" "github.com/tomochain/tomochain/eth/downloader" @@ -253,8 +253,9 @@ func testGetBlockBodies(t *testing.T, protocol int) { } // Tests that the contract codes can be retrieved based on account addresses. -func TestGetCodeLes1(t *testing.T) { testGetCode(t, 1) } func TestGetCodeLes2(t *testing.T) { testGetCode(t, 2) } +func TestGetCodeLes3(t *testing.T) { testGetCode(t, 3) } +func TestGetCodeLes4(t *testing.T) { testGetCode(t, 4) } func testGetCode(t *testing.T, protocol int) { // Assemble the test environment @@ -304,7 +305,7 @@ func testGetReceipt(t *testing.T, protocol int) { block := bc.GetBlockByNumber(i) hashes = append(hashes, block.Hash()) - receipts = append(receipts, core.GetBlockReceipts(db, block.Hash(), block.NumberU64())) + receipts = append(receipts, core.GetBlockReceipts(db, block.Hash(), block.NumberU64(), bc.Config())) } // Send the hash request and verify the response cost := peer.GetRequestCost(GetReceiptsMsg, len(hashes)) diff --git a/les/odr_test.go b/les/odr_test.go index 2f3342bbaa..ecbb69ed66 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -64,9 +64,9 @@ func odrGetBlock(ctx context.Context, db ethdb.Database, config *params.ChainCon func odrGetReceipts(ctx context.Context, db ethdb.Database, config *params.ChainConfig, bc *core.BlockChain, lc *light.LightChain, bhash common.Hash) []byte { var receipts types.Receipts if bc != nil { - receipts = core.GetBlockReceipts(db, bhash, core.GetBlockNumber(db, bhash)) + receipts = core.GetBlockReceipts(db, bhash, core.GetBlockNumber(db, bhash), config) } else { - receipts, _ = light.GetBlockReceipts(ctx, lc.Odr(), bhash, core.GetBlockNumber(db, bhash)) + receipts, _ = light.GetBlockReceipts(ctx, lc.Odr(), bhash, core.GetBlockNumber(db, bhash), config) } if receipts == nil { return nil @@ -109,12 +109,6 @@ func odrAccounts(ctx context.Context, db ethdb.Database, config *params.ChainCon // //func TestOdrContractCallLes2(t *testing.T) { testOdr(t, 2, 2, odrContractCall) } -type callmsg struct { - core.Message -} - -func (callmsg) CheckNonce() bool { return false } - func odrContractCall(ctx context.Context, db ethdb.Database, config *params.ChainConfig, bc *core.BlockChain, lc *light.LightChain, bhash common.Hash) []byte { data := common.Hex2Bytes("60CD26850000000000000000000000000000000000000000000000000000000000000000") @@ -133,20 +127,18 @@ func odrContractCall(ctx context.Context, db ethdb.Database, config *params.Chai if value, ok := feeCapacity[testContractAddr]; ok { balanceTokenFee = value } - //msg := &Message{ - // To: call.To, - // From: call.From, - // Value: call.Value, - // GasLimit: call.Gas, - // GasPrice: call.GasPrice, - // GasFeeCap: call.GasFeeCap, - // GasTipCap: call.GasTipCap, - // Data: call.Data, - // AccessList: call.AccessList, - // SkipAccountChecks: false, - //} - msg := callmsg{types.NewMessage(from.Address(), &testContractAddr, 0, new(big.Int), 100000, new(big.Int), data, false, balanceTokenFee)} - + fromAddr := from.Address() + msg := &core.Message{ + To: &fromAddr, + From: testContractAddr, + Nonce: 0, + Value: new(big.Int), + GasLimit: 100000, + GasPrice: new(big.Int), + Data: data, + SkipAccountChecks: false, + BalanceTokenFee: balanceTokenFee, + } context := core.NewEVMContext(msg, header, bc, nil) vmenv := vm.NewEVM(context, statedb, nil, config, vm.Config{}) @@ -165,7 +157,17 @@ func odrContractCall(ctx context.Context, db ethdb.Database, config *params.Chai if value, ok := feeCapacity[testContractAddr]; ok { balanceTokenFee = value } - msg := callmsg{types.NewMessage(testBankAddress, &testContractAddr, 0, new(big.Int), 100000, new(big.Int), data, false, balanceTokenFee)} + msg := &core.Message{ + To: &testBankAddress, + From: testContractAddr, + Nonce: 0, + Value: new(big.Int), + GasLimit: 100000, + GasPrice: new(big.Int), + Data: data, + SkipAccountChecks: false, + BalanceTokenFee: balanceTokenFee, + } context := core.NewEVMContext(msg, header, lc, nil) vmenv := vm.NewEVM(context, statedb, nil, config, vm.Config{}) gp := new(core.GasPool).AddGas(math.MaxUint64) diff --git a/params/config.go b/params/config.go index 41aa1748ad..8a755f178e 100644 --- a/params/config.go +++ b/params/config.go @@ -111,7 +111,7 @@ var ( // adding flags to the config to also have to set these fields. AllPosvProtocolChanges = &ChainConfig{big.NewInt(89), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, &PosvConfig{Period: 0, Epoch: 30000}} AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}, nil} - TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil, nil} + TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, big.NewInt(0), new(EthashConfig), nil, nil} TestRules = TestChainConfig.Rules(new(big.Int)) ) diff --git a/tests/block_test_util.go b/tests/block_test_util.go index b07f23b11d..e362095aa5 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -80,6 +80,7 @@ type btHeader struct { GasLimit uint64 GasUsed uint64 Timestamp *big.Int + BaseFeePerGas *big.Int } type btHeaderMarshaling struct { @@ -147,20 +148,22 @@ func (t *BlockTest) genesis(config *params.ChainConfig) *core.Genesis { Mixhash: t.json.Genesis.MixHash, Coinbase: t.json.Genesis.Coinbase, Alloc: t.json.Pre, + BaseFee: t.json.Genesis.BaseFeePerGas, } } -/* See https://github.com/ethereum/tests/wiki/Blockchain-Tests-II +/* +See https://github.com/ethereum/tests/wiki/Blockchain-Tests-II - Whether a block is valid or not is a bit subtle, it's defined by presence of - blockHeader, transactions and uncleHeaders fields. If they are missing, the block is - invalid and we must verify that we do not accept it. + Whether a block is valid or not is a bit subtle, it's defined by presence of + blockHeader, transactions and uncleHeaders fields. If they are missing, the block is + invalid and we must verify that we do not accept it. - Since some tests mix valid and invalid blocks we need to check this for every block. + Since some tests mix valid and invalid blocks we need to check this for every block. - If a block is invalid it does not necessarily fail the test, if it's invalidness is - expected we are expected to ignore it and continue processing and then validate the - post state. + If a block is invalid it does not necessarily fail the test, if it's invalidness is + expected we are expected to ignore it and continue processing and then validate the + post state. */ func (t *BlockTest) insertBlocks(blockchain *core.BlockChain) ([]btBlock, error) { validBlocks := make([]btBlock, 0) From f6d5a5836c11b72ff44bc450cf1e134ce9f27a16 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Thu, 6 Jul 2023 23:49:08 +0700 Subject: [PATCH 56/82] Fix eth and les unit tests --- core/chain_makers.go | 5 +++++ eth/handler_test.go | 20 ++++++++++---------- eth/helper_test.go | 4 ++-- les/helper_test.go | 21 ++++++++------------- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/core/chain_makers.go b/core/chain_makers.go index 40f75c1f71..136af39101 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -120,6 +120,11 @@ func (b *BlockGen) Number() *big.Int { return new(big.Int).Set(b.header.Number) } +// BaseFee returns the EIP-1559 base fee of the block being generated. +func (b *BlockGen) BaseFee() *big.Int { + return new(big.Int).Set(b.header.BaseFee) +} + // AddUncheckedReceipt forcefully adds a receipts to the block without a // backing transaction. // diff --git a/eth/handler_test.go b/eth/handler_test.go index d8d2f00979..ff1a1deb27 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -17,7 +17,6 @@ package eth import ( - "github.com/tomochain/tomochain/core/rawdb" "math" "math/big" "math/rand" @@ -27,6 +26,7 @@ import ( "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/consensus/ethash" "github.com/tomochain/tomochain/core" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/state" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/core/vm" @@ -313,13 +313,13 @@ func testGetNodeData(t *testing.T, protocol int) { switch i { case 0: // In block 1, the test bank sends account #1 some ether. - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBank), acc1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, testBankKey) + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBank), acc1Addr, big.NewInt(10_000_000_000_000_000), params.TxGas, block.BaseFee(), nil), signer, testBankKey) block.AddTx(tx) case 1: // In block 2, the test bank sends some more ether to account #1. // acc1Addr passes it on to account #2. - tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBank), acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey) - tx2, _ := types.SignTx(types.NewTransaction(block.TxNonce(acc1Addr), acc2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, acc1Key) + tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBank), acc1Addr, big.NewInt(1_000_000_000_000_000), params.TxGas, block.BaseFee(), nil), signer, testBankKey) + tx2, _ := types.SignTx(types.NewTransaction(block.TxNonce(acc1Addr), acc2Addr, big.NewInt(1_000_000_000_000_000), params.TxGas, block.BaseFee(), nil), signer, acc1Key) block.AddTx(tx1) block.AddTx(tx2) case 2: @@ -343,9 +343,9 @@ func testGetNodeData(t *testing.T, protocol int) { // Fetch for now the entire chain db hashes := []common.Hash{} - it:=db.NewIterator(nil,nil) + it := db.NewIterator(nil, nil) for it.Next() { - key:=it.Key() + key := it.Key() if len(key) == len(common.Hash{}) { hashes = append(hashes, common.BytesToHash(key)) } @@ -407,13 +407,13 @@ func testGetReceipt(t *testing.T, protocol int) { switch i { case 0: // In block 1, the test bank sends account #1 some ether. - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBank), acc1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, testBankKey) + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBank), acc1Addr, big.NewInt(10_000_000_000_000_000), params.TxGas, block.BaseFee(), nil), signer, testBankKey) block.AddTx(tx) case 1: // In block 2, the test bank sends some more ether to account #1. // acc1Addr passes it on to account #2. - tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBank), acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey) - tx2, _ := types.SignTx(types.NewTransaction(block.TxNonce(acc1Addr), acc2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, acc1Key) + tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBank), acc1Addr, big.NewInt(1_000_000_000_000_000), params.TxGas, block.BaseFee(), nil), signer, testBankKey) + tx2, _ := types.SignTx(types.NewTransaction(block.TxNonce(acc1Addr), acc2Addr, big.NewInt(1_000_000_000_000_000), params.TxGas, block.BaseFee(), nil), signer, acc1Key) block.AddTx(tx1) block.AddTx(tx2) case 2: @@ -470,7 +470,7 @@ func testDAOChallenge(t *testing.T, localForked, remoteForked bool, timeout bool var ( evmux = new(event.TypeMux) pow = ethash.NewFaker() - db = rawdb.NewMemoryDatabase() + db = rawdb.NewMemoryDatabase() config = ¶ms.ChainConfig{DAOForkBlock: big.NewInt(1), DAOForkSupport: localForked} gspec = &core.Genesis{Config: config} genesis = gspec.MustCommit(db) diff --git a/eth/helper_test.go b/eth/helper_test.go index 6ea65856f4..4cf01618c3 100644 --- a/eth/helper_test.go +++ b/eth/helper_test.go @@ -22,7 +22,6 @@ package eth import ( "crypto/ecdsa" "crypto/rand" - "github.com/tomochain/tomochain/core/rawdb" "math/big" "sort" "sync" @@ -31,6 +30,7 @@ import ( "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/consensus/ethash" "github.com/tomochain/tomochain/core" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/core/vm" "github.com/tomochain/tomochain/crypto" @@ -57,7 +57,7 @@ func newTestProtocolManager(mode downloader.SyncMode, blocks int, generator func db = rawdb.NewMemoryDatabase() gspec = &core.Genesis{ Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{testBank: {Balance: big.NewInt(1000000)}}, + Alloc: core.GenesisAlloc{testBank: {Balance: big.NewInt(100_000_000_000_000_000)}}, } genesis = gspec.MustCommit(db) blockchain, _ = core.NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}) diff --git a/les/helper_test.go b/les/helper_test.go index 67a932b4ec..2145a47f43 100644 --- a/les/helper_test.go +++ b/les/helper_test.go @@ -79,35 +79,30 @@ contract test { func testChainGen(i int, block *core.BlockGen) { signer := types.HomesteadSigner{} - switch i { case 0: // In block 1, the test bank sends account #1 some ether. - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, testBankKey) + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(10_000_000_000_000_000), params.TxGas, block.BaseFee(), nil), signer, testBankKey) block.AddTx(tx) case 1: // In block 2, the test bank sends some more ether to account #1. // acc1Addr passes it on to account #2. // acc1Addr creates a test contract. - // acc1Addr creates a test event. + tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(1_000_000_000_000_000), params.TxGas, block.BaseFee(), nil), signer, testBankKey) nonce := block.TxNonce(acc1Addr) - - tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey) - tx2, _ := types.SignTx(types.NewTransaction(nonce, acc2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, acc1Key) - tx3, _ := types.SignTx(types.NewContractCreation(nonce+1, big.NewInt(0), 200000, big.NewInt(0), testContractCode), signer, acc1Key) - testContractAddr = crypto.CreateAddress(acc1Addr, nonce+1) - tx4, _ := types.SignTx(types.NewContractCreation(nonce+2, big.NewInt(0), 200000, big.NewInt(0), testEventEmitterCode), signer, acc1Key) - testEventEmitterAddr = crypto.CreateAddress(acc1Addr, nonce+2) + tx2, _ := types.SignTx(types.NewTransaction(nonce, acc2Addr, big.NewInt(1_000_000_000_000_000), params.TxGas, block.BaseFee(), nil), signer, acc1Key) + nonce++ + tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, block.BaseFee(), testContractCode), signer, acc1Key) + testContractAddr = crypto.CreateAddress(acc1Addr, nonce) block.AddTx(tx1) block.AddTx(tx2) block.AddTx(tx3) - block.AddTx(tx4) case 2: // Block 3 is empty but was mined by account #2. block.SetCoinbase(acc2Addr) block.SetExtra([]byte("yeehaw")) data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001") - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), testContractAddr, big.NewInt(0), 100000, nil, data), signer, testBankKey) + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), testContractAddr, big.NewInt(0), 100000, block.BaseFee(), data), signer, testBankKey) block.AddTx(tx) case 3: // Block 4 includes blocks 2 and 3 as uncle headers (with modified extra data). @@ -118,7 +113,7 @@ func testChainGen(i int, block *core.BlockGen) { b3.Extra = []byte("foo") block.AddUncle(b3) data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000002") - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), testContractAddr, big.NewInt(0), 100000, nil, data), signer, testBankKey) + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), testContractAddr, big.NewInt(0), 100000, block.BaseFee(), data), signer, testBankKey) block.AddTx(tx) } } From f9db4dff43be045a7abaee8c9e5b66c2d60227c5 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Fri, 7 Jul 2023 00:16:22 +0700 Subject: [PATCH 57/82] Fix light package unit tests --- core/database_util.go | 24 ++++++++++++----- core/genesis.go | 2 +- core/headerchain.go | 12 +++++---- core/state_transition.go | 1 + light/odr_test.go | 57 +++++++++++++++++++++++++++++----------- light/trie_test.go | 9 +++++-- light/txpool_test.go | 14 ++++++---- 7 files changed, 84 insertions(+), 35 deletions(-) diff --git a/core/database_util.go b/core/database_util.go index 2afce9a124..e32274c831 100644 --- a/core/database_util.go +++ b/core/database_util.go @@ -100,16 +100,16 @@ func GetCanonicalHash(db DatabaseReader, number uint64) common.Hash { return common.BytesToHash(data) } -// missingNumber is returned by GetBlockNumber if no header with the +// MissingNumber is returned by GetBlockNumber if no header with the // given block hash has been stored in the database -const missingNumber = uint64(0xffffffffffffffff) +const MissingNumber = uint64(0xffffffffffffffff) // GetBlockNumber returns the block number assigned to a block hash // if the corresponding header is present in the database func GetBlockNumber(db DatabaseReader, hash common.Hash) uint64 { data, _ := db.Get(append(blockHashPrefix, hash.Bytes()...)) if len(data) != 8 { - return missingNumber + return MissingNumber } return binary.BigEndian.Uint64(data) } @@ -244,9 +244,10 @@ func GetBlock(db DatabaseReader, hash common.Hash, number uint64) *types.Block { return types.NewBlockWithHeader(header).WithBody(body.Transactions, body.Uncles) } -// GetBlockReceipts retrieves the receipts generated by the transactions included -// in a block given by its hash. -func GetBlockReceipts(db DatabaseReader, hash common.Hash, number uint64, config *params.ChainConfig) types.Receipts { +// ReadRawReceipts retrieves all the transaction receipts belonging to a block. +// The receipt metadata fields are not guaranteed to be populated, so they +// should not be used. Use ReadReceipts instead if the metadata is needed. +func ReadRawReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Receipts { data, _ := db.Get(append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash[:]...)) if len(data) == 0 { return nil @@ -260,6 +261,17 @@ func GetBlockReceipts(db DatabaseReader, hash common.Hash, number uint64, config for i, receipt := range storageReceipts { receipts[i] = (*types.Receipt)(receipt) } + return receipts +} + +// GetBlockReceipts retrieves the receipts generated by the transactions included +// in a block given by its hash. +func GetBlockReceipts(db DatabaseReader, hash common.Hash, number uint64, config *params.ChainConfig) types.Receipts { + // We're deriving many fields from the block body, retrieve beside the receipt + receipts := ReadRawReceipts(db, hash, number) + if receipts == nil { + return nil + } block := GetBlock(db, hash, number) if block == nil { diff --git a/core/genesis.go b/core/genesis.go index 8b8d0dd158..495c563a3b 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -198,7 +198,7 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig // Check config compatibility and write the config. Compatibility errors // are returned to the caller unless we're already at block zero. height := GetBlockNumber(db, GetHeadHeaderHash(db)) - if height == missingNumber { + if height == MissingNumber { return newcfg, stored, fmt.Errorf("missing block number for head header hash") } compatErr := storedcfg.CheckCompatible(newcfg, height) diff --git a/core/headerchain.go b/core/headerchain.go index 8365f2127d..11be19ca1f 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -26,7 +26,8 @@ import ( "sync/atomic" "time" - "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru" + "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/consensus" "github.com/tomochain/tomochain/core/types" @@ -66,9 +67,10 @@ type HeaderChain struct { } // NewHeaderChain creates a new HeaderChain structure. -// getValidator should return the parent's validator -// procInterrupt points to the parent's interrupt semaphore -// wg points to the parent's shutdown wait group +// +// getValidator should return the parent's validator +// procInterrupt points to the parent's interrupt semaphore +// wg points to the parent's shutdown wait group func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine consensus.Engine, procInterrupt func() bool) (*HeaderChain, error) { headerCache, _ := lru.New(headerCacheLimit) tdCache, _ := lru.New(tdCacheLimit) @@ -114,7 +116,7 @@ func (hc *HeaderChain) GetBlockNumber(hash common.Hash) uint64 { return cached.(uint64) } number := GetBlockNumber(hc.chainDb, hash) - if number != missingNumber { + if number != MissingNumber { hc.numberCache.Add(hash, number) } return number diff --git a/core/state_transition.go b/core/state_transition.go index 5ea25beea4..211b16a51f 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -276,6 +276,7 @@ func (st *StateTransition) preCheck() error { } // This will panic if baseFee is nil, but BaseFee presence is verified // as part of header validation. + // TODO(trinhdn2): Instead of ignoring BaseFee check of special transactions, set a proper gas limit for them if msg.GasFeeCap.Cmp(st.evm.Context.BaseFee) < 0 && !common.SpecialSMCAddressesMap[st.to().Address().String()] { return fmt.Errorf("%w: address %v, maxFeePerGas: %s baseFee: %s", ErrFeeCapTooLow, msg.From.Hex(), msg.GasFeeCap, st.evm.Context.BaseFee) diff --git a/light/odr_test.go b/light/odr_test.go index e2684051cc..05e0584c25 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -20,16 +20,16 @@ import ( "bytes" "context" "errors" - "github.com/tomochain/tomochain/consensus" - "github.com/tomochain/tomochain/core/rawdb" "math/big" "testing" "time" "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/common/math" + "github.com/tomochain/tomochain/consensus" "github.com/tomochain/tomochain/consensus/ethash" "github.com/tomochain/tomochain/core" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/state" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/core/vm" @@ -43,7 +43,7 @@ import ( var ( testBankKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) - testBankFunds = big.NewInt(100000000) + testBankFunds = big.NewInt(1_000_000_000_000_000_000) acc1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") acc2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") @@ -74,7 +74,10 @@ func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error { case *BlockRequest: req.Rlp = core.GetBodyRLP(odr.sdb, req.Hash, core.GetBlockNumber(odr.sdb, req.Hash)) case *ReceiptsRequest: - req.Receipts = core.GetBlockReceipts(odr.sdb, req.Hash, core.GetBlockNumber(odr.sdb, req.Hash)) + number := core.GetBlockNumber(odr.sdb, req.Hash) + if number != core.MissingNumber { + req.Receipts = core.ReadRawReceipts(odr.sdb, req.Hash, number) + } case *TrieRequest: t, _ := trie.New(req.Id.Root, trie.NewDatabase(odr.sdb)) nodes := NewNodeSet() @@ -110,9 +113,16 @@ func TestOdrGetReceiptsLes1(t *testing.T) { testChainOdr(t, 1, odrGetReceipts) } func odrGetReceipts(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc *LightChain, bhash common.Hash) ([]byte, error) { var receipts types.Receipts if bc != nil { - receipts = core.GetBlockReceipts(db, bhash, core.GetBlockNumber(db, bhash)) + if number := core.GetBlockNumber(db, bhash); number != core.MissingNumber { + if block := core.GetBlock(db, bhash, number); block != nil { + receipts = core.GetBlockReceipts(db, bhash, number, bc.Config()) + } + } } else { - receipts, _ = GetBlockReceipts(ctx, lc.Odr(), bhash, core.GetBlockNumber(db, bhash)) + number := core.GetBlockNumber(db, bhash) + if number != core.MissingNumber { + receipts, _ = GetBlockReceipts(ctx, lc.Odr(), bhash, number, lc.Config()) + } } if receipts == nil { return nil, nil @@ -183,7 +193,18 @@ func odrContractCall(ctx context.Context, db ethdb.Database, bc *core.BlockChain if value, ok := feeCapacity[testContractAddr]; ok { balanceTokenFee = value } - msg := callmsg{types.NewMessage(testBankAddress, &testContractAddr, 0, new(big.Int), 1000000, new(big.Int), data, false, balanceTokenFee)} + msg := &core.Message{ + From: testBankAddress, + To: &testContractAddr, + Value: new(big.Int), + GasLimit: 1000000, + GasPrice: big.NewInt(params.InitialBaseFee), + GasFeeCap: big.NewInt(params.InitialBaseFee), + GasTipCap: new(big.Int), + Data: data, + SkipAccountChecks: true, + BalanceTokenFee: balanceTokenFee, + } context := core.NewEVMContext(msg, header, chain, nil) vmenv := vm.NewEVM(context, st, nil, config, vm.Config{}) gp := new(core.GasPool).AddGas(math.MaxUint64) @@ -202,17 +223,17 @@ func testChainGen(i int, block *core.BlockGen) { switch i { case 0: // In block 1, the test bank sends account #1 some ether. - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, testBankKey) + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(10_000_000_000_000_000), params.TxGas, block.BaseFee(), nil), signer, testBankKey) block.AddTx(tx) case 1: // In block 2, the test bank sends some more ether to account #1. // acc1Addr passes it on to account #2. // acc1Addr creates a test contract. - tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey) + tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(1_000_000_000_000_000), params.TxGas, block.BaseFee(), nil), signer, testBankKey) nonce := block.TxNonce(acc1Addr) - tx2, _ := types.SignTx(types.NewTransaction(nonce, acc2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, acc1Key) + tx2, _ := types.SignTx(types.NewTransaction(nonce, acc2Addr, big.NewInt(1_000_000_000_000_000), params.TxGas, block.BaseFee(), nil), signer, acc1Key) nonce++ - tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), testContractCode), signer, acc1Key) + tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, block.BaseFee(), testContractCode), signer, acc1Key) testContractAddr = crypto.CreateAddress(acc1Addr, nonce) block.AddTx(tx1) block.AddTx(tx2) @@ -222,7 +243,7 @@ func testChainGen(i int, block *core.BlockGen) { block.SetCoinbase(acc2Addr) block.SetExtra([]byte("yeehaw")) data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001") - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), testContractAddr, big.NewInt(0), 100000, nil, data), signer, testBankKey) + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), testContractAddr, big.NewInt(0), 100000, block.BaseFee(), data), signer, testBankKey) block.AddTx(tx) case 3: // Block 4 includes blocks 2 and 3 as uncle headers (with modified extra data). @@ -233,16 +254,20 @@ func testChainGen(i int, block *core.BlockGen) { b3.Extra = []byte("foo") block.AddUncle(b3) data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000002") - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), testContractAddr, big.NewInt(0), 100000, nil, data), signer, testBankKey) + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), testContractAddr, big.NewInt(0), 100000, block.BaseFee(), data), signer, testBankKey) block.AddTx(tx) } } func testChainOdr(t *testing.T, protocol int, fn odrTestFn) { var ( - sdb = rawdb.NewMemoryDatabase() - ldb = rawdb.NewMemoryDatabase() - gspec = core.Genesis{Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}} + sdb = rawdb.NewMemoryDatabase() + ldb = rawdb.NewMemoryDatabase() + gspec = core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } genesis = gspec.MustCommit(sdb) ) gspec.MustCommit(ldb) diff --git a/light/trie_test.go b/light/trie_test.go index 53cf77dce9..978676d458 100644 --- a/light/trie_test.go +++ b/light/trie_test.go @@ -20,12 +20,13 @@ import ( "bytes" "context" "fmt" - "github.com/tomochain/tomochain/core/rawdb" + "math/big" "testing" "github.com/davecgh/go-spew/spew" "github.com/tomochain/tomochain/consensus/ethash" "github.com/tomochain/tomochain/core" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/state" "github.com/tomochain/tomochain/core/vm" "github.com/tomochain/tomochain/params" @@ -36,7 +37,11 @@ func TestNodeIterator(t *testing.T) { var ( fulldb = rawdb.NewMemoryDatabase() lightdb = rawdb.NewMemoryDatabase() - gspec = core.Genesis{Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}} + gspec = core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } genesis = gspec.MustCommit(fulldb) ) gspec.MustCommit(lightdb) diff --git a/light/txpool_test.go b/light/txpool_test.go index 97104bd683..e62c61c939 100644 --- a/light/txpool_test.go +++ b/light/txpool_test.go @@ -18,7 +18,6 @@ package light import ( "context" - "github.com/tomochain/tomochain/core/rawdb" "math" "math/big" "testing" @@ -27,6 +26,7 @@ import ( "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/consensus/ethash" "github.com/tomochain/tomochain/core" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/core/vm" "github.com/tomochain/tomochain/params" @@ -77,13 +77,17 @@ func txPoolTestChainGen(i int, block *core.BlockGen) { func TestTxPool(t *testing.T) { for i := range testTx { - testTx[i], _ = types.SignTx(types.NewTransaction(uint64(i), acc1Addr, big.NewInt(10000), params.TxGas, nil, nil), types.HomesteadSigner{}, testBankKey) + testTx[i], _ = types.SignTx(types.NewTransaction(uint64(i), acc1Addr, big.NewInt(10000), params.TxGas, big.NewInt(params.InitialBaseFee), nil), types.HomesteadSigner{}, testBankKey) } var ( - sdb = rawdb.NewMemoryDatabase() - ldb = rawdb.NewMemoryDatabase() - gspec = core.Genesis{Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}} + sdb = rawdb.NewMemoryDatabase() + ldb = rawdb.NewMemoryDatabase() + gspec = core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } genesis = gspec.MustCommit(sdb) ) gspec.MustCommit(ldb) From 9dc9ae56bb0568c3310f64fcd08762ac235ba28a Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Fri, 7 Jul 2023 00:52:25 +0700 Subject: [PATCH 58/82] Fix core package unit tests --- contracts/blocksigner/blocksigner_test.go | 2 +- core/blockchain_test.go | 75 ++++++++++++++--------- core/chain_makers.go | 5 +- core/dao_test.go | 11 +++- core/database_util.go | 10 +-- core/database_util_test.go | 58 ++++++++++++++---- internal/cmdtest/test_cmd.go | 56 ++++++++++++----- params/config.go | 2 +- 8 files changed, 155 insertions(+), 64 deletions(-) diff --git a/contracts/blocksigner/blocksigner_test.go b/contracts/blocksigner/blocksigner_test.go index 051f42f9d7..c94ab8a611 100644 --- a/contracts/blocksigner/blocksigner_test.go +++ b/contracts/blocksigner/blocksigner_test.go @@ -18,6 +18,7 @@ package blocksigner import ( "context" "math/big" + "math/rand" "testing" "time" @@ -26,7 +27,6 @@ import ( "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/core" "github.com/tomochain/tomochain/crypto" - "math/rand" ) var ( diff --git a/core/blockchain_test.go b/core/blockchain_test.go index cf209d9f99..0ddfc682f0 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -17,6 +17,7 @@ package core import ( + "errors" "fmt" "math/big" "math/rand" @@ -554,13 +555,14 @@ func TestFastVsFullChains(t *testing.T) { gendb = rawdb.NewMemoryDatabase() key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) - funds = big.NewInt(1000000000) + funds = big.NewInt(1000000000000000) gspec = &Genesis{ - Config: params.TestChainConfig, - Alloc: GenesisAlloc{address: {Balance: funds}}, + Config: params.TestChainConfig, + Alloc: GenesisAlloc{address: {Balance: funds}}, + BaseFee: big.NewInt(params.InitialBaseFee), } genesis = gspec.MustCommit(gendb) - signer = types.NewEIP155Signer(gspec.Config.ChainId) + signer = types.LatestSigner(gspec.Config) ) blocks, receipts := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), gendb, 1024, func(i int, block *BlockGen) { block.SetCoinbase(common.Address{0x00}) @@ -568,7 +570,7 @@ func TestFastVsFullChains(t *testing.T) { // If the block number is multiple of 3, send a few bonus transactions to the miner if i%3 == 2 { for j := 0; j < i%4+1; j++ { - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, nil, nil), signer, key) + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, block.header.BaseFee, nil), signer, key) if err != nil { panic(err) } @@ -729,9 +731,9 @@ func TestChainTxReorgs(t *testing.T) { Config: params.TestChainConfig, GasLimit: 3141592, Alloc: GenesisAlloc{ - addr1: {Balance: big.NewInt(1000000)}, - addr2: {Balance: big.NewInt(1000000)}, - addr3: {Balance: big.NewInt(1000000)}, + addr1: {Balance: big.NewInt(1000000000000000)}, + addr2: {Balance: big.NewInt(1000000000000000)}, + addr3: {Balance: big.NewInt(1000000000000000)}, }, } genesis = gspec.MustCommit(db) @@ -741,8 +743,8 @@ func TestChainTxReorgs(t *testing.T) { // Create two transactions shared between the chains: // - postponed: transaction included at a later block in the forked chain // - swapped: transaction included at the same block number in the forked chain - postponed, _ := types.SignTx(types.NewTransaction(0, addr1, big.NewInt(1000), params.TxGas, nil, nil), signer, key1) - swapped, _ := types.SignTx(types.NewTransaction(1, addr1, big.NewInt(1000), params.TxGas, nil, nil), signer, key1) + postponed, _ := types.SignTx(types.NewTransaction(0, addr1, big.NewInt(1000), params.TxGas, big.NewInt(params.InitialBaseFee), nil), signer, key1) + swapped, _ := types.SignTx(types.NewTransaction(1, addr1, big.NewInt(1000), params.TxGas, big.NewInt(params.InitialBaseFee), nil), signer, key1) // Create two transactions that will be dropped by the forked chain: // - pastDrop: transaction dropped retroactively from a past block @@ -758,13 +760,13 @@ func TestChainTxReorgs(t *testing.T) { chain, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 3, func(i int, gen *BlockGen) { switch i { case 0: - pastDrop, _ = types.SignTx(types.NewTransaction(gen.TxNonce(addr2), addr2, big.NewInt(1000), params.TxGas, nil, nil), signer, key2) + pastDrop, _ = types.SignTx(types.NewTransaction(gen.TxNonce(addr2), addr2, big.NewInt(1000), params.TxGas, gen.header.BaseFee, nil), signer, key2) gen.AddTx(pastDrop) // This transaction will be dropped in the fork from below the split point gen.AddTx(postponed) // This transaction will be postponed till block #3 in the fork case 2: - freshDrop, _ = types.SignTx(types.NewTransaction(gen.TxNonce(addr2), addr2, big.NewInt(1000), params.TxGas, nil, nil), signer, key2) + freshDrop, _ = types.SignTx(types.NewTransaction(gen.TxNonce(addr2), addr2, big.NewInt(1000), params.TxGas, gen.header.BaseFee, nil), signer, key2) gen.AddTx(freshDrop) // This transaction will be dropped in the fork from exactly at the split point gen.AddTx(swapped) // This transaction will be swapped out at the exact height @@ -783,18 +785,18 @@ func TestChainTxReorgs(t *testing.T) { chain, _ = GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 5, func(i int, gen *BlockGen) { switch i { case 0: - pastAdd, _ = types.SignTx(types.NewTransaction(gen.TxNonce(addr3), addr3, big.NewInt(1000), params.TxGas, nil, nil), signer, key3) + pastAdd, _ = types.SignTx(types.NewTransaction(gen.TxNonce(addr3), addr3, big.NewInt(1000), params.TxGas, gen.header.BaseFee, nil), signer, key3) gen.AddTx(pastAdd) // This transaction needs to be injected during reorg case 2: gen.AddTx(postponed) // This transaction was postponed from block #1 in the original chain gen.AddTx(swapped) // This transaction was swapped from the exact current spot in the original chain - freshAdd, _ = types.SignTx(types.NewTransaction(gen.TxNonce(addr3), addr3, big.NewInt(1000), params.TxGas, nil, nil), signer, key3) + freshAdd, _ = types.SignTx(types.NewTransaction(gen.TxNonce(addr3), addr3, big.NewInt(1000), params.TxGas, gen.header.BaseFee, nil), signer, key3) gen.AddTx(freshAdd) // This transaction will be added exactly at reorg time case 3: - futureAdd, _ = types.SignTx(types.NewTransaction(gen.TxNonce(addr3), addr3, big.NewInt(1000), params.TxGas, nil, nil), signer, key3) + futureAdd, _ = types.SignTx(types.NewTransaction(gen.TxNonce(addr3), addr3, big.NewInt(1000), params.TxGas, gen.header.BaseFee, nil), signer, key3) gen.AddTx(futureAdd) // This transaction will be added after a full reorg } }) @@ -839,9 +841,9 @@ func TestLogReorgs(t *testing.T) { db = rawdb.NewMemoryDatabase() // this code generates a log code = common.Hex2Bytes("60606040525b7f24ec1d3ff24c2f6ff210738839dbc339cd45a5294d85c79361016243157aae7b60405180905060405180910390a15b600a8060416000396000f360606040526008565b00") - gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000)}}} + gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} genesis = gspec.MustCommit(db) - signer = types.NewEIP155Signer(gspec.Config.ChainId) + signer = types.LatestSigner(gspec.Config) ) blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) @@ -851,7 +853,7 @@ func TestLogReorgs(t *testing.T) { blockchain.SubscribeRemovedLogsEvent(rmLogsCh) chain, _ := GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 2, func(i int, gen *BlockGen) { if i == 1 { - tx, err := types.SignTx(types.NewContractCreation(gen.TxNonce(addr1), new(big.Int), 1000000, new(big.Int), code), signer, key1) + tx, err := types.SignTx(types.NewContractCreation(gen.TxNonce(addr1), new(big.Int), 1000000, gen.header.BaseFee, code), signer, key1) if err != nil { t.Fatalf("failed to create tx: %v", err) } @@ -1014,8 +1016,13 @@ func TestEIP155Transition(t *testing.T) { funds = big.NewInt(1000000000) deleteAddr = common.Address{1} gspec = &Genesis{ - Config: ¶ms.ChainConfig{ChainId: big.NewInt(1), EIP155Block: big.NewInt(2), HomesteadBlock: new(big.Int)}, - Alloc: GenesisAlloc{address: {Balance: funds}, deleteAddr: {Balance: new(big.Int)}}, + Config: ¶ms.ChainConfig{ + ChainId: big.NewInt(1), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(2), + HomesteadBlock: new(big.Int), + }, + Alloc: GenesisAlloc{address: {Balance: funds}, deleteAddr: {Balance: new(big.Int)}}, } genesis = gspec.MustCommit(db) ) @@ -1104,8 +1111,8 @@ func TestEIP155Transition(t *testing.T) { } }) _, err := blockchain.InsertChain(blocks) - if err != types.ErrInvalidChainId { - t.Error("expected error:", types.ErrInvalidChainId) + if have, want := err, types.ErrInvalidChainId; !errors.Is(have, want) { + t.Errorf("have %v, want %v", have, want) } } @@ -1185,7 +1192,11 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { engine := ethash.NewFaker() db := rawdb.NewMemoryDatabase() - genesis := new(Genesis).MustCommit(db) + g := &Genesis{ + Config: params.TestChainConfig, + BaseFee: big.NewInt(params.InitialBaseFee), + } + genesis := g.MustCommit(db) blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) // Generate a bunch of fork blocks, each side forking from the canonical chain @@ -1201,7 +1212,7 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { // Import the canonical and fork chain side by side, verifying the current block // and current header consistency diskdb := rawdb.NewMemoryDatabase() - new(Genesis).MustCommit(diskdb) + g.MustCommit(diskdb) chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}) if err != nil { @@ -1230,7 +1241,11 @@ func TestTrieForkGC(t *testing.T) { engine := ethash.NewFaker() db := rawdb.NewMemoryDatabase() - genesis := new(Genesis).MustCommit(db) + g := &Genesis{ + Config: params.TestChainConfig, + BaseFee: big.NewInt(params.InitialBaseFee), + } + genesis := g.MustCommit(db) blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*triesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) // Generate a bunch of fork blocks, each side forking from the canonical chain @@ -1245,7 +1260,7 @@ func TestTrieForkGC(t *testing.T) { } // Import the canonical and fork chain side by side, forcing the trie cache to cache both diskdb := rawdb.NewMemoryDatabase() - new(Genesis).MustCommit(diskdb) + g.MustCommit(diskdb) chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}) if err != nil { @@ -1276,7 +1291,11 @@ func TestLargeReorgTrieGC(t *testing.T) { engine := ethash.NewFaker() db := rawdb.NewMemoryDatabase() - genesis := new(Genesis).MustCommit(db) + g := &Genesis{ + Config: params.TestChainConfig, + BaseFee: big.NewInt(params.InitialBaseFee), + } + genesis := g.MustCommit(db) shared, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) original, _ := GenerateChain(params.TestChainConfig, shared[len(shared)-1], engine, db, 2*triesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) }) @@ -1284,7 +1303,7 @@ func TestLargeReorgTrieGC(t *testing.T) { // Import the shared chain and the original canonical one diskdb := rawdb.NewMemoryDatabase() - new(Genesis).MustCommit(diskdb) + g.MustCommit(diskdb) chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}) if err != nil { diff --git a/core/chain_makers.go b/core/chain_makers.go index 136af39101..3499368f8c 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -299,7 +299,10 @@ func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.S // header only chain. func newCanonical(engine consensus.Engine, n int, full bool) (ethdb.Database, *BlockChain, error) { // Initialize a fresh chain with only a genesis block - gspec := new(Genesis) + gspec := &Genesis{ + BaseFee: big.NewInt(params.InitialBaseFee), + Config: params.AllEthashProtocolChanges, + } db := rawdb.NewMemoryDatabase() genesis := gspec.MustCommit(db) diff --git a/core/dao_test.go b/core/dao_test.go index 015e7db24d..d11903cb64 100644 --- a/core/dao_test.go +++ b/core/dao_test.go @@ -17,11 +17,11 @@ package core import ( - "github.com/tomochain/tomochain/core/rawdb" "math/big" "testing" "github.com/tomochain/tomochain/consensus/ethash" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/vm" "github.com/tomochain/tomochain/params" ) @@ -30,10 +30,15 @@ import ( // blocks based on their extradata fields. func TestDAOForkRangeExtradata(t *testing.T) { forkBlock := big.NewInt(32) + chainConfig := *params.TestChainConfig + chainConfig.HomesteadBlock = big.NewInt(0) - // Generate a common prefix for both pro-forkers and non-forkers db := rawdb.NewMemoryDatabase() - gspec := new(Genesis) + // Generate a common prefix for both pro-forkers and non-forkers + gspec := &Genesis{ + BaseFee: big.NewInt(params.InitialBaseFee), + Config: &chainConfig, + } genesis := gspec.MustCommit(db) prefix, _ := GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, int(forkBlock.Int64()-1), func(i int, gen *BlockGen) {}) diff --git a/core/database_util.go b/core/database_util.go index e32274c831..b1a0025ac8 100644 --- a/core/database_util.go +++ b/core/database_util.go @@ -252,7 +252,7 @@ func ReadRawReceipts(db DatabaseReader, hash common.Hash, number uint64) types.R if len(data) == 0 { return nil } - storageReceipts := []*types.ReceiptForStorage{} + var storageReceipts []*types.ReceiptForStorage if err := rlp.DecodeBytes(data, &storageReceipts); err != nil { log.Error("Invalid receipt array RLP", "hash", hash, "err", err) return nil @@ -273,9 +273,9 @@ func GetBlockReceipts(db DatabaseReader, hash common.Hash, number uint64, config return nil } - block := GetBlock(db, hash, number) - if block == nil { - log.Error("Missing block body but have receipt", "hash", hash, "number", number) + body := GetBody(db, hash, number) + if body == nil { + log.Error("Missing body but have receipt", "hash", hash, "number", number) return nil } header := GetHeader(db, hash, number) @@ -285,7 +285,7 @@ func GetBlockReceipts(db DatabaseReader, hash common.Hash, number uint64, config } else { baseFee = header.BaseFee } - if err := receipts.DeriveFields(config, hash, number, baseFee, ([]*types.Transaction)(block.Transactions())); err != nil { + if err := receipts.DeriveFields(config, hash, number, baseFee, ([]*types.Transaction)(body.Transactions)); err != nil { log.Error("Failed to derive block receipts fields", "hash", hash, "number", number, "err", err) return nil } diff --git a/core/database_util_test.go b/core/database_util_test.go index ab57e8876e..ed33f825d7 100644 --- a/core/database_util_test.go +++ b/core/database_util_test.go @@ -18,6 +18,8 @@ package core import ( "bytes" + "encoding/hex" + "fmt" "math/big" "testing" @@ -336,6 +338,11 @@ func TestLookupStorage(t *testing.T) { func TestBlockReceiptStorage(t *testing.T) { db := rawdb.NewMemoryDatabase() + // Create a live block since we need metadata to reconstruct the receipt + tx1 := types.NewTransaction(1, common.HexToAddress("0x1"), big.NewInt(1), 1, big.NewInt(1), nil) + tx2 := types.NewTransaction(2, common.HexToAddress("0x2"), big.NewInt(2), 2, big.NewInt(2), nil) + + body := &types.Body{Transactions: types.Transactions{tx1, tx2}} receipt1 := &types.Receipt{ Status: types.ReceiptStatusFailed, CumulativeGasUsed: 1, @@ -347,6 +354,7 @@ func TestBlockReceiptStorage(t *testing.T) { ContractAddress: common.BytesToAddress([]byte{0x01, 0x11, 0x11}), GasUsed: 111111, } + receipt1.Bloom = types.CreateBloom(types.Receipts{receipt1}) receipt2 := &types.Receipt{ PostState: common.Hash{2}.Bytes(), CumulativeGasUsed: 2, @@ -358,6 +366,7 @@ func TestBlockReceiptStorage(t *testing.T) { ContractAddress: common.BytesToAddress([]byte{0x02, 0x22, 0x22}), GasUsed: 222222, } + receipt2.Bloom = types.CreateBloom(types.Receipts{receipt2}) receipts := []*types.Receipt{receipt1, receipt2} // Check that no receipt entries are in a pristine database @@ -365,25 +374,52 @@ func TestBlockReceiptStorage(t *testing.T) { if rs := GetBlockReceipts(db, hash, 0, params.TestChainConfig); len(rs) != 0 { t.Fatalf("non existent receipts returned: %v", rs) } + // Insert the body that corresponds to the receipts + WriteBody(db, hash, 0, body) + // Insert the receipt slice into the database and check presence - if err := WriteBlockReceipts(db, hash, 0, receipts); err != nil { - t.Fatalf("failed to write block receipts: %v", err) - } + WriteBlockReceipts(db, hash, 0, receipts) if rs := GetBlockReceipts(db, hash, 0, params.TestChainConfig); len(rs) == 0 { t.Fatalf("no receipts returned") } else { - for i := 0; i < len(receipts); i++ { - rlpHave, _ := rlp.EncodeToBytes(rs[i]) - rlpWant, _ := rlp.EncodeToBytes(receipts[i]) - - if !bytes.Equal(rlpHave, rlpWant) { - t.Fatalf("receipt #%d: receipt mismatch: have %v, want %v", i, rs[i], receipts[i]) - } + if err := checkReceiptsRLP(rs, receipts); err != nil { + t.Fatalf(err.Error()) } } - // Delete the receipt slice and check purge + // Delete the body and ensure that the receipts are no longer returned (metadata can't be recomputed) + DeleteBody(db, hash, 0) + if rs := GetBlockReceipts(db, hash, 0, params.TestChainConfig); rs != nil { + t.Fatalf("receipts returned when body was deleted: %v", rs) + } + // Ensure that receipts without metadata can be returned without the block body too + if err := checkReceiptsRLP(ReadRawReceipts(db, hash, 0), receipts); err != nil { + t.Fatalf(err.Error()) + } + // Sanity check that body alone without the receipt is a full purge + WriteBody(db, hash, 0, body) + DeleteBlockReceipts(db, hash, 0) if rs := GetBlockReceipts(db, hash, 0, params.TestChainConfig); len(rs) != 0 { t.Fatalf("deleted receipts returned: %v", rs) } } + +func checkReceiptsRLP(have, want types.Receipts) error { + if len(have) != len(want) { + return fmt.Errorf("receipts sizes mismatch: have %d, want %d", len(have), len(want)) + } + for i := 0; i < len(want); i++ { + rlpHave, err := rlp.EncodeToBytes(have[i]) + if err != nil { + return err + } + rlpWant, err := rlp.EncodeToBytes(want[i]) + if err != nil { + return err + } + if !bytes.Equal(rlpHave, rlpWant) { + return fmt.Errorf("receipt #%d: receipt mismatch: have %s, want %s", i, hex.EncodeToString(rlpHave), hex.EncodeToString(rlpWant)) + } + } + return nil +} diff --git a/internal/cmdtest/test_cmd.go b/internal/cmdtest/test_cmd.go index c2969f8dd1..df5107952e 100644 --- a/internal/cmdtest/test_cmd.go +++ b/internal/cmdtest/test_cmd.go @@ -21,12 +21,13 @@ import ( "bytes" "fmt" "io" - "io/ioutil" "os" "os/exec" "regexp" "strings" "sync" + "sync/atomic" + "syscall" "testing" "text/template" "time" @@ -50,12 +51,17 @@ type TestCmd struct { stdout *bufio.Reader stdin io.WriteCloser stderr *testlogger + // Err will contain the process exit error or interrupt signal error + Err error } +var id int32 + // Run exec's the current binary using name as argv[0] which will trigger the // reexec init function for that name (e.g. "geth-test" in cmd/geth/run_test.go) func (tt *TestCmd) Run(name string, args ...string) { - tt.stderr = &testlogger{t: tt.T} + id := atomic.AddInt32(&id, 1) + tt.stderr = &testlogger{t: tt.T, name: fmt.Sprintf("%d", id)} tt.cmd = &exec.Cmd{ Path: reexec.Self(), Args: append([]string{name}, args...), @@ -74,10 +80,10 @@ func (tt *TestCmd) Run(name string, args ...string) { } } -// InputLine writes the given text to the childs stdin. +// InputLine writes the given text to the child's stdin. // This method can also be called from an expect template, e.g.: // -// geth.expect(`Passphrase: {{.InputLine "password"}}`) +// geth.expect(`Passphrase: {{.InputLine "password"}}`) func (tt *TestCmd) InputLine(s string) string { io.WriteString(tt.stdin, s+"\n") return "" @@ -111,6 +117,13 @@ func (tt *TestCmd) Expect(tplsource string) { tt.Logf("Matched stdout text:\n%s", want) } +// Output reads all output from stdout, and returns the data. +func (tt *TestCmd) Output() []byte { + var buf []byte + tt.withKillTimeout(func() { buf, _ = io.ReadAll(tt.stdout) }) + return buf +} + func (tt *TestCmd) matchExactOutput(want []byte) error { buf := make([]byte, len(want)) n := 0 @@ -124,12 +137,12 @@ func (tt *TestCmd) matchExactOutput(want []byte) error { // Find the mismatch position. for i := 0; i < n; i++ { if want[i] != buf[i] { - return fmt.Errorf("Output mismatch at â—Š:\n---------------- (stdout text)\n%s%s\n---------------- (expected text)\n%s", + return fmt.Errorf("output mismatch at â—Š:\n---------------- (stdout text)\n%sâ—Š%s\n---------------- (expected text)\n%s", buf[:i], buf[i:n], want) } } if n < len(want) { - return fmt.Errorf("Not enough output, got until â—Š:\n---------------- (stdout text)\n%s\n---------------- (expected text)\n%sâ—Š%s", + return fmt.Errorf("not enough output, got until â—Š:\n---------------- (stdout text)\n%s\n---------------- (expected text)\n%sâ—Š%s", buf, want[:n], want[n:]) } } @@ -170,7 +183,7 @@ func (tt *TestCmd) ExpectRegexp(regex string) (*regexp.Regexp, []string) { func (tt *TestCmd) ExpectExit() { var output []byte tt.withKillTimeout(func() { - output, _ = ioutil.ReadAll(tt.stdout) + output, _ = io.ReadAll(tt.stdout) }) tt.WaitExit() if tt.Cleanup != nil { @@ -182,11 +195,25 @@ func (tt *TestCmd) ExpectExit() { } func (tt *TestCmd) WaitExit() { - tt.cmd.Wait() + tt.Err = tt.cmd.Wait() } func (tt *TestCmd) Interrupt() { - tt.cmd.Process.Signal(os.Interrupt) + tt.Err = tt.cmd.Process.Signal(os.Interrupt) +} + +// ExitStatus exposes the process' OS exit code +// It will only return a valid value after the process has finished. +func (tt *TestCmd) ExitStatus() int { + if tt.Err != nil { + exitErr := tt.Err.(*exec.ExitError) + if exitErr != nil { + if status, ok := exitErr.Sys().(syscall.WaitStatus); ok { + return status.ExitStatus() + } + } + } + return 0 } // StderrText returns any stderr output written so far. @@ -210,7 +237,7 @@ func (tt *TestCmd) Kill() { } func (tt *TestCmd) withKillTimeout(fn func()) { - timeout := time.AfterFunc(5*time.Second, func() { + timeout := time.AfterFunc(30*time.Second, func() { tt.Log("killing the child process (timeout)") tt.Kill() }) @@ -221,16 +248,17 @@ func (tt *TestCmd) withKillTimeout(fn func()) { // testlogger logs all written lines via t.Log and also // collects them for later inspection. type testlogger struct { - t *testing.T - mu sync.Mutex - buf bytes.Buffer + t *testing.T + mu sync.Mutex + buf bytes.Buffer + name string } func (tl *testlogger) Write(b []byte) (n int, err error) { lines := bytes.Split(b, []byte("\n")) for _, line := range lines { if len(line) > 0 { - tl.t.Logf("(stderr) %s", line) + tl.t.Logf("(stderr:%v) %s", tl.name, line) } } tl.mu.Lock() diff --git a/params/config.go b/params/config.go index 8a755f178e..4ca56255d8 100644 --- a/params/config.go +++ b/params/config.go @@ -102,7 +102,7 @@ var ( // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil, nil} + AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, big.NewInt(0), new(EthashConfig), nil, nil} // AllPosvProtocolChanges contains every protocol change (EIPs) introduced // and accepted by the Ethereum core developers into the Posv consensus. From 281251e9d3cc2b6a15af38c7282bf01093e43842 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Fri, 7 Jul 2023 01:01:58 +0700 Subject: [PATCH 59/82] Fix main cmd package unit tests --- cmd/tomo/consolecmd_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/tomo/consolecmd_test.go b/cmd/tomo/consolecmd_test.go index 241373f521..894f55c698 100644 --- a/cmd/tomo/consolecmd_test.go +++ b/cmd/tomo/consolecmd_test.go @@ -52,7 +52,7 @@ func TestConsoleWelcome(t *testing.T) { tomo.SetTemplateFunc("goarch", func() string { return runtime.GOARCH }) tomo.SetTemplateFunc("gover", runtime.Version) tomo.SetTemplateFunc("tomover", func() string { return params.Version }) - tomo.SetTemplateFunc("niltime", func() string { return time.Unix(1544771829, 0).Format(time.RFC1123) }) + tomo.SetTemplateFunc("niltime", func() string { return time.Unix(1544771829, 0).Format("Mon Jan 02 2006 15:04:05 GMT-0700 (MST)") }) tomo.SetTemplateFunc("apis", func() string { return ipcAPIs }) // Verify the actual welcome message to the required template @@ -137,7 +137,7 @@ func testAttachWelcome(t *testing.T, tomo *testtomo, endpoint, apis string) { attach.SetTemplateFunc("gover", runtime.Version) attach.SetTemplateFunc("tomover", func() string { return params.Version }) attach.SetTemplateFunc("etherbase", func() string { return tomo.Etherbase }) - attach.SetTemplateFunc("niltime", func() string { return time.Unix(1544771829, 0).Format(time.RFC1123) }) + attach.SetTemplateFunc("niltime", func() string { return time.Unix(1544771829, 0).Format("Mon Jan 02 2006 15:04:05 GMT-0700 (MST)") }) attach.SetTemplateFunc("ipc", func() bool { return strings.HasPrefix(endpoint, "ipc") }) attach.SetTemplateFunc("datadir", func() string { return tomo.Datadir }) attach.SetTemplateFunc("apis", func() string { return apis }) From 4e9ac23896439fe2f4215435eac04fd5a27df4b3 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Fri, 7 Jul 2023 11:29:00 +0700 Subject: [PATCH 60/82] Fix unit tests cont. --- accounts/abi/bind/util_test.go | 5 +- accounts/keystore/keystore.go | 2 +- eth/filters/filter_system_test.go | 22 +++--- eth/filters/filter_test.go | 114 +++++++++++++++++++++--------- internal/guide/guide_test.go | 3 +- p2p/discv5/node_test.go | 2 +- 6 files changed, 101 insertions(+), 47 deletions(-) diff --git a/accounts/abi/bind/util_test.go b/accounts/abi/bind/util_test.go index 0a3290873e..a6515a4e9f 100644 --- a/accounts/abi/bind/util_test.go +++ b/accounts/abi/bind/util_test.go @@ -54,11 +54,12 @@ var waitDeployedTests = map[string]struct { func TestWaitDeployed(t *testing.T) { for name, test := range waitDeployedTests { backend := backends.NewSimulatedBackend(core.GenesisAlloc{ - crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000)}, + crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000000000)}, }) // Create the transaction. - tx := types.NewContractCreation(0, big.NewInt(0), test.gas, big.NewInt(1), common.FromHex(test.code)) + head, _ := backend.HeaderByNumber(context.Background(), nil) // Should be child's, good enough + tx := types.NewContractCreation(0, big.NewInt(0), test.gas, new(big.Int).Add(head.BaseFee, big.NewInt(1)), common.FromHex(test.code)) tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey) // Wait for it to get mined in the background. diff --git a/accounts/keystore/keystore.go b/accounts/keystore/keystore.go index 4011061b96..cf671ef300 100644 --- a/accounts/keystore/keystore.go +++ b/accounts/keystore/keystore.go @@ -309,7 +309,7 @@ func (ks *KeyStore) SignTxWithPassphrase(a accounts.Account, passphrase string, if chainID != nil { return types.SignTx(tx, types.NewEIP155Signer(chainID), key.PrivateKey) } - return types.SignTx(tx, types.HomesteadSigner{}, key.PrivateKey) + return types.SignTx(tx, types.LatestSignerForChainID(chainID), key.PrivateKey) } // Unlock unlocks the given account indefinitely. diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index eb3e7cce4b..5aeddde39d 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -145,15 +145,19 @@ func TestBlockSubscription(t *testing.T) { t.Parallel() var ( - mux = new(event.TypeMux) - db = rawdb.NewMemoryDatabase() - txFeed = new(event.Feed) - rmLogsFeed = new(event.Feed) - logsFeed = new(event.Feed) - chainFeed = new(event.Feed) - backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false) - genesis = new(core.Genesis).MustCommit(db) + mux = new(event.TypeMux) + db = rawdb.NewMemoryDatabase() + txFeed = new(event.Feed) + rmLogsFeed = new(event.Feed) + logsFeed = new(event.Feed) + chainFeed = new(event.Feed) + backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} + api = NewPublicFilterAPI(backend, false) + g = &core.Genesis{ + Config: params.TestChainConfig, + BaseFee: big.NewInt(params.InitialBaseFee), + } + genesis = g.MustCommit(db) chain, _ = core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 10, func(i int, gen *core.BlockGen) {}) chainEvents = []core.ChainEvent{} ) diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index bdfb6e37f8..6b74b3e4bd 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -18,15 +18,17 @@ package filters import ( "context" - "github.com/tomochain/tomochain/core/rawdb" "io/ioutil" "math/big" "os" + "strings" "testing" + "github.com/tomochain/tomochain/accounts/abi" "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/consensus/ethash" "github.com/tomochain/tomochain/core" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/crypto" "github.com/tomochain/tomochain/event" @@ -50,7 +52,7 @@ func BenchmarkFilters(b *testing.B) { defer os.RemoveAll(dir) var ( - db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0,"") + db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0, "") mux = new(event.TypeMux) txFeed = new(event.Feed) rmLogsFeed = new(event.Feed) @@ -115,7 +117,7 @@ func TestFilters(t *testing.T) { defer os.RemoveAll(dir) var ( - db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0,"") + db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0, "") mux = new(event.TypeMux) txFeed = new(event.Feed) rmLogsFeed = new(event.Feed) @@ -124,6 +126,12 @@ func TestFilters(t *testing.T) { backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr = crypto.PubkeyToAddress(key1.PublicKey) + signer = types.NewLondonSigner(big.NewInt(1)) + + // Logging contract + contract = common.Address{0xfe} + contract2 = common.Address{0xff} + abiStr = `[{"inputs":[],"name":"log0","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"uint256","name":"t1","type":"uint256"}],"name":"log1","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"uint256","name":"t1","type":"uint256"},{"internalType":"uint256","name":"t2","type":"uint256"}],"name":"log2","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"uint256","name":"t1","type":"uint256"},{"internalType":"uint256","name":"t2","type":"uint256"},{"internalType":"uint256","name":"t3","type":"uint256"}],"name":"log3","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"uint256","name":"t1","type":"uint256"},{"internalType":"uint256","name":"t2","type":"uint256"},{"internalType":"uint256","name":"t3","type":"uint256"},{"internalType":"uint256","name":"t4","type":"uint256"}],"name":"log4","outputs":[],"stateMutability":"nonpayable","type":"function"}]` hash1 = common.BytesToHash([]byte("topic1")) hash2 = common.BytesToHash([]byte("topic2")) @@ -132,45 +140,85 @@ func TestFilters(t *testing.T) { ) defer db.Close() - genesis := core.GenesisBlockForTesting(db, addr, big.NewInt(1000000)) - chain, receipts := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 1000, func(i int, gen *core.BlockGen) { + g := &core.Genesis{ + Config: params.TestChainConfig, + BaseFee: big.NewInt(params.InitialBaseFee), + Alloc: core.GenesisAlloc{addr: {Balance: big.NewInt(0).Mul(big.NewInt(100), big.NewInt(params.Ether))}}, + } + + contractABI, err := abi.JSON(strings.NewReader(abiStr)) + if err != nil { + t.Fatal(err) + } + + // Hack: GenerateChainWithGenesis creates a new db. + // Commit the genesis manually and use GenerateChain. + _, err = g.Commit(db) + if err != nil { + t.Fatal(err) + } + chain, receipts := core.GenerateChain(g.Config, g.ToBlock(db), ethash.NewFaker(), db, 1000, func(i int, gen *core.BlockGen) { switch i { case 1: - receipt := types.NewReceipt(nil, false, 0) - receipt.Logs = []*types.Log{ - { - Address: addr, - Topics: []common.Hash{hash1}, - }, + data, err := contractABI.Pack("log1", hash1.Big()) + if err != nil { + t.Fatal(err) } - gen.AddUncheckedReceipt(receipt) + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 0, + GasPrice: gen.BaseFee(), + Gas: 30000, + To: &contract, + Data: data, + }), signer, key1) + gen.AddTx(tx) + tx2, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 1, + GasPrice: gen.BaseFee(), + Gas: 30000, + To: &contract2, + Data: data, + }), signer, key1) + gen.AddTx(tx2) case 2: - receipt := types.NewReceipt(nil, false, 0) - receipt.Logs = []*types.Log{ - { - Address: addr, - Topics: []common.Hash{hash2}, - }, + data, err := contractABI.Pack("log2", hash2.Big(), hash1.Big()) + if err != nil { + t.Fatal(err) } - gen.AddUncheckedReceipt(receipt) + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 2, + GasPrice: gen.BaseFee(), + Gas: 30000, + To: &contract, + Data: data, + }), signer, key1) + gen.AddTx(tx) case 998: - receipt := types.NewReceipt(nil, false, 0) - receipt.Logs = []*types.Log{ - { - Address: addr, - Topics: []common.Hash{hash3}, - }, + data, err := contractABI.Pack("log1", hash3.Big()) + if err != nil { + t.Fatal(err) } - gen.AddUncheckedReceipt(receipt) + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 3, + GasPrice: gen.BaseFee(), + Gas: 30000, + To: &contract2, + Data: data, + }), signer, key1) + gen.AddTx(tx) case 999: - receipt := types.NewReceipt(nil, false, 0) - receipt.Logs = []*types.Log{ - { - Address: addr, - Topics: []common.Hash{hash4}, - }, + data, err := contractABI.Pack("log1", hash4.Big()) + if err != nil { + t.Fatal(err) } - gen.AddUncheckedReceipt(receipt) + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 4, + GasPrice: gen.BaseFee(), + Gas: 30000, + To: &contract, + Data: data, + }), signer, key1) + gen.AddTx(tx) } }) for i, block := range chain { diff --git a/internal/guide/guide_test.go b/internal/guide/guide_test.go index 8b82725811..1dfc794184 100644 --- a/internal/guide/guide_test.go +++ b/internal/guide/guide_test.go @@ -31,6 +31,7 @@ import ( "time" "github.com/tomochain/tomochain/accounts/keystore" + "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/core/types" ) @@ -75,7 +76,7 @@ func TestAccountManagement(t *testing.T) { if err != nil { t.Fatalf("Failed to create signer account: %v", err) } - tx, chain := new(types.Transaction), big.NewInt(1) + tx, chain := types.NewTransaction(0, common.Address{}, big.NewInt(0), 0, big.NewInt(0), nil), big.NewInt(1) // Sign a transaction with a single authorization if _, err := ks.SignTxWithPassphrase(signer, "Signer password", tx, chain); err != nil { diff --git a/p2p/discv5/node_test.go b/p2p/discv5/node_test.go index a28f298252..d0fa6880a3 100644 --- a/p2p/discv5/node_test.go +++ b/p2p/discv5/node_test.go @@ -141,7 +141,7 @@ var parseNodeTests = []struct { { // This test checks that errors from url.Parse are handled. rawurl: "://foo", - wantError: `parse ://foo: missing protocol scheme`, + wantError: `parse "://foo": missing protocol scheme`, }, } From cb7f8eee08af954a18d7860a9c458af62d2fac23 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Fri, 7 Jul 2023 12:00:59 +0700 Subject: [PATCH 61/82] Fix unit tests cont. --- p2p/discover/node_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/discover/node_test.go b/p2p/discover/node_test.go index 8e3da2c2aa..ddf8a7bd98 100644 --- a/p2p/discover/node_test.go +++ b/p2p/discover/node_test.go @@ -142,7 +142,7 @@ var parseNodeTests = []struct { { // This test checks that errors from url.Parse are handled. rawurl: "://foo", - wantError: `parse ://foo: missing protocol scheme`, + wantError: `parse "://foo": missing protocol scheme`, }, } From dbcb9fbb2f90b026ca12942781d788d86f31602b Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Fri, 7 Jul 2023 17:15:12 +0700 Subject: [PATCH 62/82] [WIP] Update simulated backend to test special transactions --- accounts/abi/bind/auth.go | 14 +++ accounts/abi/bind/backends/simulated.go | 117 +++++++++++++++++++--- cmd/puppeth/wizard_genesis.go | 17 ++-- contracts/blocksigner/blocksigner_test.go | 12 ++- tests/state_test.go | 3 + tests/transaction_test.go | 3 + 6 files changed, 143 insertions(+), 23 deletions(-) diff --git a/accounts/abi/bind/auth.go b/accounts/abi/bind/auth.go index 12b1607a93..171b1ce235 100644 --- a/accounts/abi/bind/auth.go +++ b/accounts/abi/bind/auth.go @@ -51,6 +51,20 @@ func NewTransactor(keyin io.Reader, passphrase string) (*TransactOpts, error) { return NewKeyedTransactor(key.PrivateKey), nil } +// NewTransactorWithChainID is a utility method to easily create a transaction signer from +// an encrypted json key stream and the associated passphrase. +func NewTransactorWithChainID(keyin io.Reader, passphrase string, chainID *big.Int) (*TransactOpts, error) { + json, err := io.ReadAll(keyin) + if err != nil { + return nil, err + } + key, err := keystore.DecryptKey(json, passphrase) + if err != nil { + return nil, err + } + return NewKeyedTransactorWithChainID(key.PrivateKey, chainID) +} + // NewKeyedTransactor is a utility method to easily create a transaction signer // from a single private key. // diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 7bebd34984..397caa3427 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -47,8 +47,11 @@ import ( // This nil assignment ensures compile time that SimulatedBackend implements bind.ContractBackend. var _ bind.ContractBackend = (*SimulatedBackend)(nil) -var errBlockNumberUnsupported = errors.New("SimulatedBackend cannot access blocks other than the latest block") -var errGasEstimationFailed = errors.New("gas required exceeds allowance or always failing transaction") +var ( + errBlockNumberUnsupported = errors.New("SimulatedBackend cannot access blocks other than the latest block") + errBlockDoesNotExist = errors.New("block does not exist in blockchain") + errGasEstimationFailed = errors.New("gas required exceeds allowance or always failing transaction") +) // SimulatedBackend implements bind.ContractBackend, simulating a blockchain in // the background. Its main purpose is to allow easily testing contract bindings. @@ -58,7 +61,7 @@ type SimulatedBackend struct { mu sync.Mutex pendingBlock *types.Block // Currently pending block that will be imported on request - pendingState *state.StateDB // Currently pending state that will be the active on on request + pendingState *state.StateDB // Currently pending state that will be the active on request events *filters.EventSystem // Event system for filtering log events live @@ -77,9 +80,13 @@ func NewSimulatedBackend(alloc core.GenesisAlloc) *SimulatedBackend { database: database, blockchain: blockchain, config: genesis.Config, - events: filters.NewEventSystem(new(event.TypeMux), &filterBackend{database, blockchain}, false), } - backend.rollback() + backend.events = filters.NewEventSystem(new(event.TypeMux), &filterBackend{database, blockchain, backend}, false) + + header := backend.blockchain.CurrentBlock() + block := backend.blockchain.GetBlock(header.Hash(), header.Number().Uint64()) + + backend.rollback(block) return backend } @@ -92,7 +99,9 @@ func (b *SimulatedBackend) Commit() { if _, err := b.blockchain.InsertChain([]*types.Block{b.pendingBlock}); err != nil { panic(err) // This cannot happen unless the simulator is wrong, fail in that case } - b.rollback() + // Using the last inserted block here makes it possible to build on a side + // chain after a fork. + b.rollback(b.pendingBlock) } // Rollback aborts all pending transactions, reverting to the last committed state. @@ -100,15 +109,17 @@ func (b *SimulatedBackend) Rollback() { b.mu.Lock() defer b.mu.Unlock() - b.rollback() + header := b.blockchain.CurrentBlock() + block := b.blockchain.GetBlock(header.Hash(), header.Number().Uint64()) + + b.rollback(block) } -func (b *SimulatedBackend) rollback() { - blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), ethash.NewFaker(), b.database, 1, func(int, *core.BlockGen) {}) - statedb, _ := b.blockchain.State() +func (b *SimulatedBackend) rollback(parent *types.Block) { + blocks, _ := core.GenerateChain(b.config, parent, ethash.NewFaker(), b.database, 1, func(int, *core.BlockGen) {}) b.pendingBlock = blocks[0] - b.pendingState, _ = state.New(b.pendingBlock.Root(), statedb.Database()) + b.pendingState, _ = state.New(b.pendingBlock.Root(), state.NewDatabase(b.database)) } // CodeAt returns the code associated with a certain account in the blockchain. @@ -179,6 +190,69 @@ func (b *SimulatedBackend) TransactionReceipt(ctx context.Context, txHash common return receipt, nil } +// BlockByHash retrieves a block based on the block hash. +func (b *SimulatedBackend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) { + b.mu.Lock() + defer b.mu.Unlock() + + return b.blockByHash(ctx, hash) +} + +// blockByHash retrieves a block based on the block hash without Locking. +func (b *SimulatedBackend) blockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) { + if hash == b.pendingBlock.Hash() { + return b.pendingBlock, nil + } + + block := b.blockchain.GetBlockByHash(hash) + if block != nil { + return block, nil + } + + return nil, errBlockDoesNotExist +} + +// BlockByNumber retrieves a block from the database by number, caching it +// (associated with its hash) if found. +func (b *SimulatedBackend) BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) { + b.mu.Lock() + defer b.mu.Unlock() + + return b.blockByNumber(ctx, number) +} + +// blockByNumber retrieves a block from the database by number, caching it +// (associated with its hash) if found without Lock. +func (b *SimulatedBackend) blockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) { + if number == nil || number.Cmp(b.pendingBlock.Number()) == 0 { + return b.blockByHash(ctx, b.blockchain.CurrentBlock().Hash()) + } + + block := b.blockchain.GetBlockByNumber(uint64(number.Int64())) + if block == nil { + return nil, errBlockDoesNotExist + } + + return block, nil +} + +// HeaderByHash returns a block header from the current canonical chain. +func (b *SimulatedBackend) HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) { + b.mu.Lock() + defer b.mu.Unlock() + + if hash == b.pendingBlock.Hash() { + return b.pendingBlock.Header(), nil + } + + header := b.blockchain.GetHeaderByHash(hash) + if header == nil { + return nil, errBlockDoesNotExist + } + + return header, nil +} + // HeaderByNumber returns a block header from the current canonical chain. If number is // nil, the latest known header is returned. func (b *SimulatedBackend) HeaderByNumber(ctx context.Context, block *big.Int) (*types.Header, error) { @@ -450,7 +524,14 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa b.mu.Lock() defer b.mu.Unlock() - sender, err := types.Sender(types.HomesteadSigner{}, tx) + // Get the last block + block, err := b.blockByHash(ctx, b.pendingBlock.ParentHash()) + if err != nil { + return errors.New("could not fetch parent") + } + // Check transaction validity + signer := types.MakeSigner(b.blockchain.Config(), block.Number()) + sender, err := types.Sender(signer, tx) if err != nil { panic(fmt.Errorf("invalid transaction: %v", err)) } @@ -487,7 +568,7 @@ func (b *SimulatedBackend) FilterLogs(ctx context.Context, query tomochain.Filte to = query.ToBlock.Int64() } // Construct and execute the filter - filter := filters.New(&filterBackend{b.database, b.blockchain}, from, to, query.Addresses, query.Topics) + filter := filters.New(&filterBackend{b.database, b.blockchain, b}, from, to, query.Addresses, query.Topics) logs, err := filter.Logs(ctx) if err != nil { @@ -552,11 +633,17 @@ func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error { return nil } +// Blockchain returns the underlying blockchain. +func (b *SimulatedBackend) Blockchain() *core.BlockChain { + return b.blockchain +} + // filterBackend implements filters.Backend to support filtering for logs without // taking bloom-bits acceleration structures into account. type filterBackend struct { - db ethdb.Database - bc *core.BlockChain + db ethdb.Database + bc *core.BlockChain + backend *SimulatedBackend } func (fb *filterBackend) ChainDb() ethdb.Database { return fb.db } diff --git a/cmd/puppeth/wizard_genesis.go b/cmd/puppeth/wizard_genesis.go index f8f423c11b..6b8baf071c 100644 --- a/cmd/puppeth/wizard_genesis.go +++ b/cmd/puppeth/wizard_genesis.go @@ -21,7 +21,6 @@ import ( "encoding/json" "fmt" "io/ioutil" - "math/rand" "time" "github.com/tomochain/tomochain/common" @@ -31,6 +30,7 @@ import ( "context" "math/big" + "math/rand" "github.com/tomochain/tomochain/accounts/abi/bind" "github.com/tomochain/tomochain/accounts/abi/bind/backends" @@ -80,6 +80,12 @@ func (w *wizard) makeGenesis() { Period: 15, Epoch: 30000, } + + // Query the user for some custom extras + fmt.Println() + fmt.Println("Specify your chain/network ID if you want an explicit one (default = random)") + genesis.Config.ChainId = new(big.Int).SetUint64(uint64(w.readDefaultInt(rand.Intn(65536)))) + fmt.Println() fmt.Println("How many seconds should blocks take? (default = 15)") genesis.Config.Clique.Period = uint64(w.readDefaultInt(15)) @@ -179,7 +185,10 @@ func (w *wizard) makeGenesis() { pKey, _ := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr := crypto.PubkeyToAddress(pKey.PublicKey) contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(1000000000)}}) - transactOpts := bind.NewKeyedTransactor(pKey) + transactOpts, err := bind.NewKeyedTransactorWithChainID(pKey, genesis.Config.ChainId) + if err != nil { + fmt.Println("Can't create TransactionOpts:", err) + } validatorAddress, _, err := validatorContract.DeployValidator(transactOpts, contractBackend, signers, validatorCaps, owner) if err != nil { @@ -342,10 +351,6 @@ func (w *wizard) makeGenesis() { for i := int64(0); i < 2; i++ { genesis.Alloc[common.BigToAddress(big.NewInt(i))] = core.GenesisAccount{Balance: big.NewInt(0)} } - // Query the user for some custom extras - fmt.Println() - fmt.Println("Specify your chain/network ID if you want an explicit one (default = random)") - genesis.Config.ChainId = new(big.Int).SetUint64(uint64(w.readDefaultInt(rand.Intn(65536)))) // All done, store the genesis and flush to disk log.Info("Configured new genesis block") diff --git a/contracts/blocksigner/blocksigner_test.go b/contracts/blocksigner/blocksigner_test.go index c94ab8a611..31b591527a 100644 --- a/contracts/blocksigner/blocksigner_test.go +++ b/contracts/blocksigner/blocksigner_test.go @@ -35,8 +35,16 @@ var ( ) func TestBlockSigner(t *testing.T) { - contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(1000000000)}}) - transactOpts := bind.NewKeyedTransactor(key) + contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{ + addr: { + Balance: big.NewInt(10_000_000_000_000_000), + Nonce: 1, + }, + }) + transactOpts, err := bind.NewKeyedTransactorWithChainID(key, contractBackend.Blockchain().Config().ChainId) + if err != nil { + t.Fatalf("can't create TransactOpts: %v", err) + } blockSignerAddress, blockSigner, err := DeployBlockSigner(transactOpts, contractBackend, big.NewInt(99)) if err != nil { diff --git a/tests/state_test.go b/tests/state_test.go index 7c8c5e9268..d7e01df561 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -26,6 +26,9 @@ import ( ) func TestState(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } t.Parallel() st := new(testMatcher) diff --git a/tests/transaction_test.go b/tests/transaction_test.go index fee0dd3b23..b08d80157f 100644 --- a/tests/transaction_test.go +++ b/tests/transaction_test.go @@ -24,6 +24,9 @@ import ( ) func TestTransaction(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } t.Parallel() txt := new(testMatcher) From 127fa6ce6b2e1e8600a73dd289831f8a93f8595f Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Sat, 8 Jul 2023 00:01:10 +0700 Subject: [PATCH 63/82] [WIP] Special contracts test units --- accounts/abi/bind/backends/simulated.go | 7 ++++- contracts/blocksigner/blocksigner_test.go | 1 - contracts/chequebook/cheque.go | 17 ++++++++-- contracts/chequebook/cheque_test.go | 18 ++++++----- contracts/ens/ens_test.go | 7 +++-- contracts/randomize/randomize_test.go | 23 ++++++++++---- contracts/tests/Inherited_test.go | 12 ++++--- contracts/trc21issuer/trc21issuer_test.go | 15 ++++++--- contracts/utils.go | 4 +-- contracts/validator/validator_test.go | 38 +++++++++++++++++------ core/blockchain.go | 10 +++++- core/genesis.go | 5 ++- eth/downloader/downloader_test.go | 24 +++++++------- swarm/services/swap/swap.go | 8 ++--- 14 files changed, 133 insertions(+), 56 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 397caa3427..a17495b261 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -72,7 +72,12 @@ type SimulatedBackend struct { // for testing purposes. func NewSimulatedBackend(alloc core.GenesisAlloc) *SimulatedBackend { database := rawdb.NewMemoryDatabase() - genesis := core.Genesis{Config: params.AllEthashProtocolChanges, Alloc: alloc, GasLimit: 42000000} + genesis := core.Genesis{ + Config: params.AllEthashProtocolChanges, + Alloc: alloc, + GasLimit: 42000000, + BaseFee: big.NewInt(params.InitialBaseFee), + } genesis.MustCommit(database) blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}) diff --git a/contracts/blocksigner/blocksigner_test.go b/contracts/blocksigner/blocksigner_test.go index 31b591527a..0b70519ed8 100644 --- a/contracts/blocksigner/blocksigner_test.go +++ b/contracts/blocksigner/blocksigner_test.go @@ -38,7 +38,6 @@ func TestBlockSigner(t *testing.T) { contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{ addr: { Balance: big.NewInt(10_000_000_000_000_000), - Nonce: 1, }, }) transactOpts, err := bind.NewKeyedTransactorWithChainID(key, contractBackend.Blockchain().Config().ChainId) diff --git a/contracts/chequebook/cheque.go b/contracts/chequebook/cheque.go index bd5088d6e7..55c26c18ff 100644 --- a/contracts/chequebook/cheque.go +++ b/contracts/chequebook/cheque.go @@ -36,6 +36,10 @@ import ( "sync" "time" + "github.com/tomochain/tomochain/params" + + "github.com/tomochain/tomochain/core" + "github.com/tomochain/tomochain/accounts/abi/bind" "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/common/hexutil" @@ -65,6 +69,7 @@ type Backend interface { bind.ContractBackend TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) BalanceAt(ctx context.Context, address common.Address, blockNum *big.Int) (*big.Int, error) + Blockchain() *core.BlockChain } // Cheque represents a payment promise to a single beneficiary. @@ -122,7 +127,10 @@ func NewChequebook(path string, contractAddr common.Address, prvKey *ecdsa.Priva if err != nil { return nil, err } - transactOpts := bind.NewKeyedTransactor(prvKey) + transactOpts, err := bind.NewKeyedTransactorWithChainID(prvKey, backend.Blockchain().Config().ChainId) + if err != nil { + return nil, err + } session := &contract.ChequebookSession{ Contract: chbook, TransactOpts: *transactOpts, @@ -445,7 +453,7 @@ type Inbox struct { // NewInbox creates an Inbox. An Inboxes is not persisted, the cumulative sum is updated // from blockchain when first cheque is received. -func NewInbox(prvKey *ecdsa.PrivateKey, contractAddr, beneficiary common.Address, signer *ecdsa.PublicKey, abigen bind.ContractBackend) (self *Inbox, err error) { +func NewInbox(prvKey *ecdsa.PrivateKey, contractAddr, beneficiary common.Address, signer *ecdsa.PublicKey, abigen bind.ContractBackend, chainConfig *params.ChainConfig) (self *Inbox, err error) { if signer == nil { return nil, fmt.Errorf("signer is null") } @@ -453,7 +461,10 @@ func NewInbox(prvKey *ecdsa.PrivateKey, contractAddr, beneficiary common.Address if err != nil { return nil, err } - transactOpts := bind.NewKeyedTransactor(prvKey) + transactOpts, err := bind.NewKeyedTransactorWithChainID(prvKey, chainConfig.ChainId) + if err != nil { + return nil, err + } transactOpts.GasLimit = gasToCash session := &contract.ChequebookSession{ Contract: chbook, diff --git a/contracts/chequebook/cheque_test.go b/contracts/chequebook/cheque_test.go index 3461cda03c..78e5cfbc12 100644 --- a/contracts/chequebook/cheque_test.go +++ b/contracts/chequebook/cheque_test.go @@ -43,15 +43,19 @@ var ( func newTestBackend() *backends.SimulatedBackend { return backends.NewSimulatedBackend(core.GenesisAlloc{ - addr0: {Balance: big.NewInt(1000000000)}, - addr1: {Balance: big.NewInt(1000000000)}, - addr2: {Balance: big.NewInt(1000000000)}, + addr0: {Balance: big.NewInt(10_000_000_000_000_000)}, + addr1: {Balance: big.NewInt(10_000_000_000_000_000)}, + addr2: {Balance: big.NewInt(10_000_000_000_000_000)}, }) } func deploy(prvKey *ecdsa.PrivateKey, amount *big.Int, backend *backends.SimulatedBackend) (common.Address, error) { - deployTransactor := bind.NewKeyedTransactor(prvKey) + deployTransactor, err := bind.NewKeyedTransactorWithChainID(prvKey, backend.Blockchain().Config().ChainId) + if err != nil { + return common.Address{}, err + } deployTransactor.Value = amount + deployTransactor.GasLimit = 1000000 addr, _, _, err := contract.DeployChequebook(deployTransactor, backend) if err != nil { return common.Address{}, err @@ -92,7 +96,7 @@ func TestIssueAndReceive(t *testing.T) { t.Errorf("expected: %v, got %v", "0", chbook.Balance()) } - chbox, err := NewInbox(key1, addr0, addr1, &key0.PublicKey, backend) + chbox, err := NewInbox(key1, addr0, addr1, &key0.PublicKey, backend, backend.Blockchain().Config()) if err != nil { t.Fatalf("expected no error, got %v", err) } @@ -170,7 +174,7 @@ func TestVerifyErrors(t *testing.T) { t.Fatalf("expected no error, got %v", err) } - chbox, err := NewInbox(key1, contr0, addr1, &key0.PublicKey, backend) + chbox, err := NewInbox(key1, contr0, addr1, &key0.PublicKey, backend, backend.Blockchain().Config()) if err != nil { t.Fatalf("expected no error, got %v", err) } @@ -375,7 +379,7 @@ func TestCash(t *testing.T) { t.Fatalf("expected no error, got %v", err) } backend.Commit() - chbox, err := NewInbox(key1, contr0, addr1, &key0.PublicKey, backend) + chbox, err := NewInbox(key1, contr0, addr1, &key0.PublicKey, backend, backend.Blockchain().Config()) if err != nil { t.Fatalf("expected no error, got %v", err) } diff --git a/contracts/ens/ens_test.go b/contracts/ens/ens_test.go index acd32eb9d2..657069044d 100644 --- a/contracts/ens/ens_test.go +++ b/contracts/ens/ens_test.go @@ -35,8 +35,11 @@ var ( ) func TestENS(t *testing.T) { - contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(1000000000)}}) - transactOpts := bind.NewKeyedTransactor(key) + contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(10_000_000_000_000_000)}}) + transactOpts, err := bind.NewKeyedTransactorWithChainID(key, contractBackend.Blockchain().Config().ChainId) + if err != nil { + t.Fatalf("can't create TransactOpts: %v", err) + } ensAddr, ens, err := DeployENS(transactOpts, contractBackend) if err != nil { diff --git a/contracts/randomize/randomize_test.go b/contracts/randomize/randomize_test.go index 651ee9d823..ead67f7fa0 100644 --- a/contracts/randomize/randomize_test.go +++ b/contracts/randomize/randomize_test.go @@ -28,6 +28,7 @@ import ( "github.com/tomochain/tomochain/core" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/crypto" + "github.com/tomochain/tomochain/params" ) var ( @@ -40,8 +41,15 @@ var ( ) func TestRandomize(t *testing.T) { - contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(100000000000000)}}) - transactOpts := bind.NewKeyedTransactor(key) + contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{ + addr: { + Balance: big.NewInt(10_000_000_000_000_000), + }, + }) + transactOpts, err := bind.NewKeyedTransactorWithChainID(key, contractBackend.Blockchain().Config().ChainId) + if err != nil { + t.Fatalf("can't create TransactOpts: %v", err) + } transactOpts.GasLimit = 1000000 randomizeAddress, randomize, err := DeployRandomize(transactOpts, contractBackend) @@ -70,13 +78,16 @@ func TestRandomize(t *testing.T) { } func TestSendTxRandomizeSecretAndOpening(t *testing.T) { - genesis := core.GenesisAlloc{acc1Addr: {Balance: big.NewInt(1000000000000)}} + genesis := core.GenesisAlloc{acc1Addr: {Balance: big.NewInt(100_000_000_000_000_000)}} backend := backends.NewSimulatedBackend(genesis) backend.Commit() - signer := types.HomesteadSigner{} + signer := types.LatestSignerForChainID(backend.Blockchain().Config().ChainId) ctx := context.Background() - transactOpts := bind.NewKeyedTransactor(acc1Key) + transactOpts, err := bind.NewKeyedTransactorWithChainID(acc1Key, backend.Blockchain().Config().ChainId) + if err != nil { + t.Fatalf("can't create TransactOpts: %v", err) + } transactOpts.GasLimit = 4200000 epocNumber := uint64(900) randomizeAddr, randomizeContract, err := DeployRandomize(transactOpts, backend) @@ -138,7 +149,7 @@ func TestSendTxRandomizeSecretAndOpening(t *testing.T) { t.Error("Can't decrypt secret and opening", err) } default: - tx, err := types.SignTx(types.NewTransaction(nonce, common.Address{}, new(big.Int), 21000, new(big.Int), nil), signer, acc1Key) + tx, err := types.SignTx(types.NewTransaction(nonce, common.Address{}, new(big.Int), 21000, big.NewInt(params.InitialBaseFee), nil), signer, acc1Key) if err != nil { t.Fatalf("Can't sign tx randomize: %v", err) } diff --git a/contracts/tests/Inherited_test.go b/contracts/tests/Inherited_test.go index 5640d42657..b529523b4d 100644 --- a/contracts/tests/Inherited_test.go +++ b/contracts/tests/Inherited_test.go @@ -2,15 +2,16 @@ package tests import ( "fmt" + "math/big" + "os" + "testing" + "github.com/tomochain/tomochain/accounts/abi/bind" "github.com/tomochain/tomochain/accounts/abi/bind/backends" "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/core" "github.com/tomochain/tomochain/crypto" "github.com/tomochain/tomochain/log" - "math/big" - "os" - "testing" ) var ( @@ -27,7 +28,10 @@ func TestPriceFeed(t *testing.T) { contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{ mainAddr: {Balance: big.NewInt(0).Mul(big.NewInt(10000000000000), big.NewInt(10000000000000))}, }) - transactOpts := bind.NewKeyedTransactor(mainKey) + transactOpts, err := bind.NewKeyedTransactorWithChainID(mainKey, contractBackend.Blockchain().Config().ChainId) + if err != nil { + t.Fatalf("can't create TransactOpts: %v", err) + } // deploy payer swap SMC addr, contract, err := DeployMyInherited(transactOpts, contractBackend) if err != nil { diff --git a/contracts/trc21issuer/trc21issuer_test.go b/contracts/trc21issuer/trc21issuer_test.go index 08ba96c1d0..b7ad37eda1 100644 --- a/contracts/trc21issuer/trc21issuer_test.go +++ b/contracts/trc21issuer/trc21issuer_test.go @@ -1,13 +1,14 @@ package trc21issuer import ( + "math/big" + "testing" + "github.com/tomochain/tomochain/accounts/abi/bind" "github.com/tomochain/tomochain/accounts/abi/bind/backends" "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/core" "github.com/tomochain/tomochain/crypto" - "math/big" - "testing" ) var ( @@ -32,7 +33,10 @@ func TestFeeTxWithTRC21Token(t *testing.T) { contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{ mainAddr: {Balance: big.NewInt(0).Mul(big.NewInt(10000000000000), big.NewInt(10000000000000))}, }) - transactOpts := bind.NewKeyedTransactor(mainKey) + transactOpts, err := bind.NewKeyedTransactorWithChainID(mainKey, contractBackend.Blockchain().Config().ChainId) + if err != nil { + t.Fatalf("can't create TransactOpts: %v", err) + } // deploy payer swap SMC trc21IssuerAddr, trc21Issuer, err := DeployTRC21Issuer(transactOpts, contractBackend, minApply) @@ -105,7 +109,10 @@ func TestFeeTxWithTRC21Token(t *testing.T) { } // access to address which received token trc21 but dont have tomo - key1TransactOpts := bind.NewKeyedTransactor(airdropKey) + key1TransactOpts, err := bind.NewKeyedTransactorWithChainID(airdropKey, contractBackend.Blockchain().Config().ChainId) + if err != nil { + t.Fatalf("can't create TransactOpts: %v", err) + } key1Trc20, _ := NewTRC21(key1TransactOpts, trc21TokenAddr, contractBackend) transferAmount := big.NewInt(100000) diff --git a/contracts/utils.go b/contracts/utils.go index 551e0be9cd..9f75c9779d 100644 --- a/contracts/utils.go +++ b/contracts/utils.go @@ -200,7 +200,7 @@ func BuildTxSecretRandomize(nonce uint64, randomizeAddr common.Address, epocNumb encryptSecret := Encrypt(randomizeKey, new(big.Int).SetInt64(secret).String()) inputData = append(inputData, common.LeftPadBytes([]byte(encryptSecret), int(sizeOfArray))...) } - tx := types.NewTransaction(nonce, randomizeAddr, big.NewInt(0), 200000, big.NewInt(0), inputData) + tx := types.NewTransaction(nonce, randomizeAddr, big.NewInt(0), 200000, big.NewInt(params.InitialBaseFee), inputData) return tx, nil } @@ -209,7 +209,7 @@ func BuildTxSecretRandomize(nonce uint64, randomizeAddr common.Address, epocNumb func BuildTxOpeningRandomize(nonce uint64, randomizeAddr common.Address, randomizeKey []byte) (*types.Transaction, error) { data := common.Hex2Bytes(common.HexSetOpening) inputData := append(data, randomizeKey...) - tx := types.NewTransaction(nonce, randomizeAddr, big.NewInt(0), 200000, big.NewInt(0), inputData) + tx := types.NewTransaction(nonce, randomizeAddr, big.NewInt(0), 200000, big.NewInt(params.InitialBaseFee), inputData) return tx, nil } diff --git a/contracts/validator/validator_test.go b/contracts/validator/validator_test.go index c7a452d751..4dcf2997e1 100644 --- a/contracts/validator/validator_test.go +++ b/contracts/validator/validator_test.go @@ -46,8 +46,15 @@ var ( ) func TestValidator(t *testing.T) { - contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(1000000000)}}) - transactOpts := bind.NewKeyedTransactor(key) + contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{ + addr: { + Balance: big.NewInt(10_000_000_000_000_000), + }, + }) + transactOpts, err := bind.NewKeyedTransactorWithChainID(key, contractBackend.Blockchain().Config().ChainId) + if err != nil { + t.Fatalf("can't create TransactOpts: %v", err) + } validatorCap := new(big.Int) validatorCap.SetString("50000000000000000000000", 10) @@ -83,14 +90,24 @@ func TestValidator(t *testing.T) { func TestRewardBalance(t *testing.T) { contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{ - acc1Addr: {Balance: new(big.Int).SetUint64(10000000)}, - acc2Addr: {Balance: new(big.Int).SetUint64(10000000)}, - acc4Addr: {Balance: new(big.Int).SetUint64(10000000)}, + acc1Addr: {Balance: new(big.Int).SetUint64(10_000_000_000_000_000)}, + acc2Addr: {Balance: new(big.Int).SetUint64(10_000_000_000_000_000)}, + acc4Addr: {Balance: new(big.Int).SetUint64(10_000_000_000_000_000)}, }) - acc1Opts := bind.NewKeyedTransactor(acc1Key) - acc2Opts := bind.NewKeyedTransactor(acc2Key) + + acc1Opts, err := bind.NewKeyedTransactorWithChainID(acc1Key, contractBackend.Blockchain().Config().ChainId) + if err != nil { + t.Fatalf("can't create TransactOpts: %v", err) + } + acc2Opts, err := bind.NewKeyedTransactorWithChainID(acc2Key, contractBackend.Blockchain().Config().ChainId) + if err != nil { + t.Fatalf("can't create TransactOpts: %v", err) + } accounts := []*bind.TransactOpts{acc1Opts, acc2Opts} - transactOpts := bind.NewKeyedTransactor(acc1Key) + transactOpts, err := bind.NewKeyedTransactorWithChainID(acc1Key, contractBackend.Blockchain().Config().ChainId) + if err != nil { + t.Fatalf("can't create TransactOpts: %v", err) + } // validatorAddr, _, baseValidator, err := contract.DeployTomoValidator(transactOpts, contractBackend, big.NewInt(50000), big.NewInt(99), big.NewInt(100), big.NewInt(100)) validatorCap := new(big.Int) @@ -113,7 +130,10 @@ func TestRewardBalance(t *testing.T) { contractBackend.Commit() // Propose master node acc3Addr. - opts := bind.NewKeyedTransactor(acc4Key) + opts, err := bind.NewKeyedTransactorWithChainID(acc4Key, contractBackend.Blockchain().Config().ChainId) + if err != nil { + t.Fatalf("can't create TransactOpts: %v", err) + } opts.Value = new(big.Int).SetUint64(50000) acc4Validator, _ := NewValidator(opts, validatorAddr, contractBackend) acc4Validator.Propose(acc3Addr) diff --git a/core/blockchain.go b/core/blockchain.go index 27c5e0554b..4a86dcbc41 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -58,6 +58,7 @@ var ( blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil) CheckpointCh = make(chan int) ErrNoGenesis = errors.New("Genesis not found in chain") + errChainStopped = errors.New("blockchain is stopped") ) const ( @@ -1390,6 +1391,10 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { // only reason this method exists as a separate one is to make locking cleaner // with deferred statements. func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*types.Log, error) { + // Sanity check that we have something meaningful to import + if len(chain) == 0 { + return 0, nil, nil, nil + } engine, _ := bc.Engine().(*posv.Posv) // Do a sanity check that the provided chain is actually ordered and linked @@ -1407,7 +1412,10 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty bc.wg.Add(1) defer bc.wg.Done() - bc.chainmu.Lock() + // Pre-checks passed, start the full block imports + if !bc.chainmu.TryLock() { + return 0, nil, nil, errChainStopped + } defer bc.chainmu.Unlock() // A queued approach to delivering events. This is generally diff --git a/core/genesis.go b/core/genesis.go index 495c563a3b..bd336398aa 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -313,7 +313,10 @@ func (g *Genesis) MustCommit(db ethdb.Database) *types.Block { // GenesisBlockForTesting creates and writes a block in which addr has the given wei balance. func GenesisBlockForTesting(db ethdb.Database, addr common.Address, balance *big.Int) *types.Block { - g := Genesis{Alloc: GenesisAlloc{addr: {Balance: balance}}} + g := Genesis{ + Alloc: GenesisAlloc{addr: {Balance: balance}}, + BaseFee: new(big.Int).SetUint64(params.InitialBaseFee), + } return g.MustCommit(db) } diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index af39f9856b..ae993b26bd 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -19,7 +19,6 @@ package downloader import ( "errors" "fmt" - "github.com/tomochain/tomochain/core/rawdb" "math/big" "sync" "sync/atomic" @@ -29,6 +28,7 @@ import ( "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/consensus/ethash" "github.com/tomochain/tomochain/core" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/crypto" "github.com/tomochain/tomochain/ethdb" @@ -94,7 +94,7 @@ func newTester() *downloadTester { peerChainTds: make(map[string]map[common.Hash]*big.Int), peerMissingStates: make(map[string]map[common.Hash]bool), } - tester.stateDb= rawdb.NewMemoryDatabase() + tester.stateDb = rawdb.NewMemoryDatabase() tester.stateDb.Put(genesis.Root().Bytes(), []byte{0x00}) tester.downloader = New(FullSync, tester.stateDb, new(event.TypeMux), tester, nil, tester.dropPeer) @@ -118,7 +118,7 @@ func (dl *downloadTester) makeChain(n int, seed byte, parent *types.Block, paren // If the block number is multiple of 3, send a bonus transaction to the miner if parent == dl.genesis && i%3 == 0 { signer := types.MakeSigner(params.TestChainConfig, block.Number()) - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, nil, nil), signer, testKey) + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, block.BaseFee(), nil), signer, testKey) if err != nil { panic(err) } @@ -663,12 +663,14 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng // Tests that simple synchronization against a canonical chain works correctly. // In this test common ancestor lookup should be short circuited and not require // binary searching. -func TestCanonicalSynchronisation62(t *testing.T) { testCanonicalSynchronisation(t, 62, FullSync) } -func TestCanonicalSynchronisation63Full(t *testing.T) { testCanonicalSynchronisation(t, 63, FullSync) } -func TestCanonicalSynchronisation63Fast(t *testing.T) { testCanonicalSynchronisation(t, 63, FastSync) } -func TestCanonicalSynchronisation64Full(t *testing.T) { testCanonicalSynchronisation(t, 64, FullSync) } -func TestCanonicalSynchronisation64Fast(t *testing.T) { testCanonicalSynchronisation(t, 64, FastSync) } -func TestCanonicalSynchronisation64Light(t *testing.T) { testCanonicalSynchronisation(t, 64, LightSync) } +func TestCanonicalSynchronisation62(t *testing.T) { testCanonicalSynchronisation(t, 62, FullSync) } +func TestCanonicalSynchronisation63Full(t *testing.T) { testCanonicalSynchronisation(t, 63, FullSync) } +func TestCanonicalSynchronisation63Fast(t *testing.T) { testCanonicalSynchronisation(t, 63, FastSync) } +func TestCanonicalSynchronisation64Full(t *testing.T) { testCanonicalSynchronisation(t, 64, FullSync) } +func TestCanonicalSynchronisation64Fast(t *testing.T) { testCanonicalSynchronisation(t, 64, FastSync) } +func TestCanonicalSynchronisation64Light(t *testing.T) { + testCanonicalSynchronisation(t, 64, LightSync) +} func testCanonicalSynchronisation(t *testing.T, protocol int, mode SyncMode) { t.Parallel() @@ -1357,8 +1359,8 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol int) { } } -//Tests that synchronisation progress (origin block number, current block number -//and highest block number) is tracked and updated correctly. +// Tests that synchronisation progress (origin block number, current block number +// and highest block number) is tracked and updated correctly. func TestSyncProgress62(t *testing.T) { testSyncProgress(t, 62, FullSync) } func TestSyncProgress63Full(t *testing.T) { testSyncProgress(t, 63, FullSync) } func TestSyncProgress63Fast(t *testing.T) { testSyncProgress(t, 63, FastSync) } diff --git a/swarm/services/swap/swap.go b/swarm/services/swap/swap.go index dfd0e12cfa..56c6e3bece 100644 --- a/swarm/services/swap/swap.go +++ b/swarm/services/swap/swap.go @@ -80,7 +80,7 @@ type PayProfile struct { lock sync.RWMutex } -//create params with default values +// create params with default values func NewDefaultSwapParams() *SwapParams { return &SwapParams{ PayProfile: &PayProfile{}, @@ -102,8 +102,8 @@ func NewDefaultSwapParams() *SwapParams { } } -//this can only finally be set after all config options (file, cmd line, env vars) -//have been evaluated +// this can only finally be set after all config options (file, cmd line, env vars) +// have been evaluated func (self *SwapParams) Init(contract common.Address, prvkey *ecdsa.PrivateKey) { pubkey := &prvkey.PublicKey @@ -142,7 +142,7 @@ func NewSwap(local *SwapParams, remote *SwapProfile, backend chequebook.Backend, log.Info(fmt.Sprintf("invalid contract %v for peer %v: %v)", remote.Contract.Hex()[:8], proto, err)) } else { // remote contract valid, create inbox - in, err = chequebook.NewInbox(local.privateKey, remote.Contract, local.Beneficiary, crypto.ToECDSAPub(common.FromHex(remote.PublicKey)), backend) + in, err = chequebook.NewInbox(local.privateKey, remote.Contract, local.Beneficiary, crypto.ToECDSAPub(common.FromHex(remote.PublicKey)), backend, backend.Blockchain().Config()) if err != nil { log.Warn(fmt.Sprintf("unable to set up inbox for chequebook contract %v for peer %v: %v)", remote.Contract.Hex()[:8], proto, err)) } From d5beefc7757e7a54e61ffc416ac9fac694d6995c Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Sat, 8 Jul 2023 00:20:16 +0700 Subject: [PATCH 64/82] [WIP] Special contracts test units --- accounts/abi/bind/backends/simulated.go | 6 +++--- contracts/blocksigner/blocksigner_test.go | 6 +++++- contracts/chequebook/cheque.go | 16 +++++++------- contracts/chequebook/cheque_test.go | 26 +++++++++++++++++++---- contracts/ens/ens_test.go | 7 +++++- contracts/randomize/randomize_test.go | 14 +++++++++--- contracts/tests/Inherited_test.go | 7 +++++- contracts/trc21issuer/trc21issuer_test.go | 9 ++++++-- contracts/validator/validator_test.go | 18 +++++++++++----- ethclient/ethclient.go | 10 +++++++++ swarm/services/swap/swap.go | 6 +++++- 11 files changed, 96 insertions(+), 29 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index a17495b261..8fb4f0f5c4 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -638,9 +638,9 @@ func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error { return nil } -// Blockchain returns the underlying blockchain. -func (b *SimulatedBackend) Blockchain() *core.BlockChain { - return b.blockchain +// ChainID returns the underlying blockchain. +func (b *SimulatedBackend) ChainID(ctx context.Context) (*big.Int, error) { + return b.blockchain.Config().ChainId, nil } // filterBackend implements filters.Backend to support filtering for logs without diff --git a/contracts/blocksigner/blocksigner_test.go b/contracts/blocksigner/blocksigner_test.go index 0b70519ed8..23b859cb14 100644 --- a/contracts/blocksigner/blocksigner_test.go +++ b/contracts/blocksigner/blocksigner_test.go @@ -40,7 +40,11 @@ func TestBlockSigner(t *testing.T) { Balance: big.NewInt(10_000_000_000_000_000), }, }) - transactOpts, err := bind.NewKeyedTransactorWithChainID(key, contractBackend.Blockchain().Config().ChainId) + chainID, err := contractBackend.ChainID(context.Background()) + if err != nil { + t.Fatalf("can't get chainID: %v", err) + } + transactOpts, err := bind.NewKeyedTransactorWithChainID(key, chainID) if err != nil { t.Fatalf("can't create TransactOpts: %v", err) } diff --git a/contracts/chequebook/cheque.go b/contracts/chequebook/cheque.go index 55c26c18ff..71bcf8002a 100644 --- a/contracts/chequebook/cheque.go +++ b/contracts/chequebook/cheque.go @@ -36,10 +36,6 @@ import ( "sync" "time" - "github.com/tomochain/tomochain/params" - - "github.com/tomochain/tomochain/core" - "github.com/tomochain/tomochain/accounts/abi/bind" "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/common/hexutil" @@ -69,7 +65,7 @@ type Backend interface { bind.ContractBackend TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) BalanceAt(ctx context.Context, address common.Address, blockNum *big.Int) (*big.Int, error) - Blockchain() *core.BlockChain + ChainID(ctx context.Context) (*big.Int, error) } // Cheque represents a payment promise to a single beneficiary. @@ -127,7 +123,11 @@ func NewChequebook(path string, contractAddr common.Address, prvKey *ecdsa.Priva if err != nil { return nil, err } - transactOpts, err := bind.NewKeyedTransactorWithChainID(prvKey, backend.Blockchain().Config().ChainId) + chainID, err := backend.ChainID(context.Background()) + if err != nil { + return nil, err + } + transactOpts, err := bind.NewKeyedTransactorWithChainID(prvKey, chainID) if err != nil { return nil, err } @@ -453,7 +453,7 @@ type Inbox struct { // NewInbox creates an Inbox. An Inboxes is not persisted, the cumulative sum is updated // from blockchain when first cheque is received. -func NewInbox(prvKey *ecdsa.PrivateKey, contractAddr, beneficiary common.Address, signer *ecdsa.PublicKey, abigen bind.ContractBackend, chainConfig *params.ChainConfig) (self *Inbox, err error) { +func NewInbox(prvKey *ecdsa.PrivateKey, contractAddr, beneficiary common.Address, signer *ecdsa.PublicKey, abigen bind.ContractBackend, chainID *big.Int) (self *Inbox, err error) { if signer == nil { return nil, fmt.Errorf("signer is null") } @@ -461,7 +461,7 @@ func NewInbox(prvKey *ecdsa.PrivateKey, contractAddr, beneficiary common.Address if err != nil { return nil, err } - transactOpts, err := bind.NewKeyedTransactorWithChainID(prvKey, chainConfig.ChainId) + transactOpts, err := bind.NewKeyedTransactorWithChainID(prvKey, chainID) if err != nil { return nil, err } diff --git a/contracts/chequebook/cheque_test.go b/contracts/chequebook/cheque_test.go index 78e5cfbc12..4cea8be78d 100644 --- a/contracts/chequebook/cheque_test.go +++ b/contracts/chequebook/cheque_test.go @@ -17,6 +17,7 @@ package chequebook import ( + "context" "crypto/ecdsa" "math/big" "os" @@ -50,7 +51,11 @@ func newTestBackend() *backends.SimulatedBackend { } func deploy(prvKey *ecdsa.PrivateKey, amount *big.Int, backend *backends.SimulatedBackend) (common.Address, error) { - deployTransactor, err := bind.NewKeyedTransactorWithChainID(prvKey, backend.Blockchain().Config().ChainId) + chainID, err := backend.ChainID(context.Background()) + if err != nil { + return common.Address{}, err + } + deployTransactor, err := bind.NewKeyedTransactorWithChainID(prvKey, chainID) if err != nil { return common.Address{}, err } @@ -96,7 +101,11 @@ func TestIssueAndReceive(t *testing.T) { t.Errorf("expected: %v, got %v", "0", chbook.Balance()) } - chbox, err := NewInbox(key1, addr0, addr1, &key0.PublicKey, backend, backend.Blockchain().Config()) + chainID, err := backend.ChainID(context.Background()) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + chbox, err := NewInbox(key1, addr0, addr1, &key0.PublicKey, backend, chainID) if err != nil { t.Fatalf("expected no error, got %v", err) } @@ -174,7 +183,11 @@ func TestVerifyErrors(t *testing.T) { t.Fatalf("expected no error, got %v", err) } - chbox, err := NewInbox(key1, contr0, addr1, &key0.PublicKey, backend, backend.Blockchain().Config()) + chainID, err := backend.ChainID(context.Background()) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + chbox, err := NewInbox(key1, contr0, addr1, &key0.PublicKey, backend, chainID) if err != nil { t.Fatalf("expected no error, got %v", err) } @@ -379,7 +392,12 @@ func TestCash(t *testing.T) { t.Fatalf("expected no error, got %v", err) } backend.Commit() - chbox, err := NewInbox(key1, contr0, addr1, &key0.PublicKey, backend, backend.Blockchain().Config()) + + chainID, err := backend.ChainID(context.Background()) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + chbox, err := NewInbox(key1, contr0, addr1, &key0.PublicKey, backend, chainID) if err != nil { t.Fatalf("expected no error, got %v", err) } diff --git a/contracts/ens/ens_test.go b/contracts/ens/ens_test.go index 657069044d..bfb2959a12 100644 --- a/contracts/ens/ens_test.go +++ b/contracts/ens/ens_test.go @@ -17,6 +17,7 @@ package ens import ( + "context" "math/big" "testing" @@ -36,7 +37,11 @@ var ( func TestENS(t *testing.T) { contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(10_000_000_000_000_000)}}) - transactOpts, err := bind.NewKeyedTransactorWithChainID(key, contractBackend.Blockchain().Config().ChainId) + chainID, err := contractBackend.ChainID(context.Background()) + if err != nil { + t.Fatalf("can't get chainID: %v", err) + } + transactOpts, err := bind.NewKeyedTransactorWithChainID(key, chainID) if err != nil { t.Fatalf("can't create TransactOpts: %v", err) } diff --git a/contracts/randomize/randomize_test.go b/contracts/randomize/randomize_test.go index ead67f7fa0..87320dde05 100644 --- a/contracts/randomize/randomize_test.go +++ b/contracts/randomize/randomize_test.go @@ -46,7 +46,11 @@ func TestRandomize(t *testing.T) { Balance: big.NewInt(10_000_000_000_000_000), }, }) - transactOpts, err := bind.NewKeyedTransactorWithChainID(key, contractBackend.Blockchain().Config().ChainId) + chainID, err := contractBackend.ChainID(context.Background()) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + transactOpts, err := bind.NewKeyedTransactorWithChainID(key, chainID) if err != nil { t.Fatalf("can't create TransactOpts: %v", err) } @@ -81,10 +85,14 @@ func TestSendTxRandomizeSecretAndOpening(t *testing.T) { genesis := core.GenesisAlloc{acc1Addr: {Balance: big.NewInt(100_000_000_000_000_000)}} backend := backends.NewSimulatedBackend(genesis) backend.Commit() - signer := types.LatestSignerForChainID(backend.Blockchain().Config().ChainId) + chainID, err := backend.ChainID(context.Background()) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + signer := types.LatestSignerForChainID(chainID) ctx := context.Background() - transactOpts, err := bind.NewKeyedTransactorWithChainID(acc1Key, backend.Blockchain().Config().ChainId) + transactOpts, err := bind.NewKeyedTransactorWithChainID(acc1Key, chainID) if err != nil { t.Fatalf("can't create TransactOpts: %v", err) } diff --git a/contracts/tests/Inherited_test.go b/contracts/tests/Inherited_test.go index b529523b4d..90dbf4c0c1 100644 --- a/contracts/tests/Inherited_test.go +++ b/contracts/tests/Inherited_test.go @@ -1,6 +1,7 @@ package tests import ( + "context" "fmt" "math/big" "os" @@ -28,7 +29,11 @@ func TestPriceFeed(t *testing.T) { contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{ mainAddr: {Balance: big.NewInt(0).Mul(big.NewInt(10000000000000), big.NewInt(10000000000000))}, }) - transactOpts, err := bind.NewKeyedTransactorWithChainID(mainKey, contractBackend.Blockchain().Config().ChainId) + chainID, err := contractBackend.ChainID(context.Background()) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + transactOpts, err := bind.NewKeyedTransactorWithChainID(mainKey, chainID) if err != nil { t.Fatalf("can't create TransactOpts: %v", err) } diff --git a/contracts/trc21issuer/trc21issuer_test.go b/contracts/trc21issuer/trc21issuer_test.go index b7ad37eda1..b808279a63 100644 --- a/contracts/trc21issuer/trc21issuer_test.go +++ b/contracts/trc21issuer/trc21issuer_test.go @@ -1,6 +1,7 @@ package trc21issuer import ( + "context" "math/big" "testing" @@ -33,7 +34,11 @@ func TestFeeTxWithTRC21Token(t *testing.T) { contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{ mainAddr: {Balance: big.NewInt(0).Mul(big.NewInt(10000000000000), big.NewInt(10000000000000))}, }) - transactOpts, err := bind.NewKeyedTransactorWithChainID(mainKey, contractBackend.Blockchain().Config().ChainId) + chainID, err := contractBackend.ChainID(context.Background()) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + transactOpts, err := bind.NewKeyedTransactorWithChainID(mainKey, chainID) if err != nil { t.Fatalf("can't create TransactOpts: %v", err) } @@ -109,7 +114,7 @@ func TestFeeTxWithTRC21Token(t *testing.T) { } // access to address which received token trc21 but dont have tomo - key1TransactOpts, err := bind.NewKeyedTransactorWithChainID(airdropKey, contractBackend.Blockchain().Config().ChainId) + key1TransactOpts, err := bind.NewKeyedTransactorWithChainID(airdropKey, chainID) if err != nil { t.Fatalf("can't create TransactOpts: %v", err) } diff --git a/contracts/validator/validator_test.go b/contracts/validator/validator_test.go index 4dcf2997e1..74d9c2ddad 100644 --- a/contracts/validator/validator_test.go +++ b/contracts/validator/validator_test.go @@ -51,7 +51,11 @@ func TestValidator(t *testing.T) { Balance: big.NewInt(10_000_000_000_000_000), }, }) - transactOpts, err := bind.NewKeyedTransactorWithChainID(key, contractBackend.Blockchain().Config().ChainId) + chainID, err := contractBackend.ChainID(context.Background()) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + transactOpts, err := bind.NewKeyedTransactorWithChainID(key, chainID) if err != nil { t.Fatalf("can't create TransactOpts: %v", err) } @@ -95,16 +99,20 @@ func TestRewardBalance(t *testing.T) { acc4Addr: {Balance: new(big.Int).SetUint64(10_000_000_000_000_000)}, }) - acc1Opts, err := bind.NewKeyedTransactorWithChainID(acc1Key, contractBackend.Blockchain().Config().ChainId) + chainID, err := contractBackend.ChainID(context.Background()) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + acc1Opts, err := bind.NewKeyedTransactorWithChainID(acc1Key, chainID) if err != nil { t.Fatalf("can't create TransactOpts: %v", err) } - acc2Opts, err := bind.NewKeyedTransactorWithChainID(acc2Key, contractBackend.Blockchain().Config().ChainId) + acc2Opts, err := bind.NewKeyedTransactorWithChainID(acc2Key, chainID) if err != nil { t.Fatalf("can't create TransactOpts: %v", err) } accounts := []*bind.TransactOpts{acc1Opts, acc2Opts} - transactOpts, err := bind.NewKeyedTransactorWithChainID(acc1Key, contractBackend.Blockchain().Config().ChainId) + transactOpts, err := bind.NewKeyedTransactorWithChainID(acc1Key, chainID) if err != nil { t.Fatalf("can't create TransactOpts: %v", err) } @@ -130,7 +138,7 @@ func TestRewardBalance(t *testing.T) { contractBackend.Commit() // Propose master node acc3Addr. - opts, err := bind.NewKeyedTransactorWithChainID(acc4Key, contractBackend.Blockchain().Config().ChainId) + opts, err := bind.NewKeyedTransactorWithChainID(acc4Key, chainID) if err != nil { t.Fatalf("can't create TransactOpts: %v", err) } diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 4f868b4fc0..b59d36b3ca 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -53,6 +53,16 @@ func NewClient(c *rpc.Client) *Client { // Blockchain Access +// ChainID retrieves the current chain ID for transaction replay protection. +func (ec *Client) ChainID(ctx context.Context) (*big.Int, error) { + var result hexutil.Big + err := ec.c.CallContext(ctx, &result, "eth_chainId") + if err != nil { + return nil, err + } + return (*big.Int)(&result), err +} + // BlockByHash returns the given full block. // // Note that loading full blocks requires two requests. Use HeaderByHash diff --git a/swarm/services/swap/swap.go b/swarm/services/swap/swap.go index 56c6e3bece..7abf68d0a3 100644 --- a/swarm/services/swap/swap.go +++ b/swarm/services/swap/swap.go @@ -142,7 +142,11 @@ func NewSwap(local *SwapParams, remote *SwapProfile, backend chequebook.Backend, log.Info(fmt.Sprintf("invalid contract %v for peer %v: %v)", remote.Contract.Hex()[:8], proto, err)) } else { // remote contract valid, create inbox - in, err = chequebook.NewInbox(local.privateKey, remote.Contract, local.Beneficiary, crypto.ToECDSAPub(common.FromHex(remote.PublicKey)), backend, backend.Blockchain().Config()) + chainID, err := backend.ChainID(ctx) + if err != nil { + log.Warn(fmt.Sprintf("unable to get chainID for peer %v: %v)", proto, err)) + } + in, err = chequebook.NewInbox(local.privateKey, remote.Contract, local.Beneficiary, crypto.ToECDSAPub(common.FromHex(remote.PublicKey)), backend, chainID) if err != nil { log.Warn(fmt.Sprintf("unable to set up inbox for chequebook contract %v for peer %v: %v)", remote.Contract.Hex()[:8], proto, err)) } From 482df9a524f36973d6be3367878965ac7889e994 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Sat, 8 Jul 2023 02:54:17 +0700 Subject: [PATCH 65/82] [WIP] Fix eth downloader unit tests --- accounts/abi/bind/base.go | 2 +- contracts/chequebook/cheque.go | 9 +- contracts/randomize/randomize_test.go | 11 +- contracts/tests/Inherited_test.go | 8 +- eth/downloader/downloader.go | 40 +- eth/downloader/downloader_test.go | 650 ++++++++++++-------------- eth/downloader/queue.go | 1 + eth/downloader/testchain_test.go | 230 +++++++++ params/network_params.go | 12 + 9 files changed, 573 insertions(+), 390 deletions(-) create mode 100644 eth/downloader/testchain_test.go diff --git a/accounts/abi/bind/base.go b/accounts/abi/bind/base.go index 8d5b31cd59..4fc30e778a 100644 --- a/accounts/abi/bind/base.go +++ b/accounts/abi/bind/base.go @@ -134,7 +134,7 @@ func (c *BoundContract) Call(opts *CallOpts, result interface{}, method string, return err } var ( - msg = tomochain.CallMsg{From: opts.From, To: &c.address, Data: input, GasPrice: common.MinGasPrice, Gas: uint64(4200000)} + msg = tomochain.CallMsg{From: opts.From, To: &c.address, Data: input} ctx = ensureContext(opts.Context) code []byte output []byte diff --git a/contracts/chequebook/cheque.go b/contracts/chequebook/cheque.go index 71bcf8002a..fd4b12f997 100644 --- a/contracts/chequebook/cheque.go +++ b/contracts/chequebook/cheque.go @@ -344,7 +344,14 @@ func (self *Chequebook) Deposit(amount *big.Int) (string, error) { // The caller must hold self.lock. func (self *Chequebook) deposit(amount *big.Int) (string, error) { // since the amount is variable here, we do not use sessions - depositTransactor := bind.NewKeyedTransactor(self.prvKey) + chainID, err := self.backend.ChainID(context.Background()) + if err != nil { + return "", err + } + depositTransactor, err := bind.NewKeyedTransactorWithChainID(self.prvKey, chainID) + if err != nil { + return "", err + } depositTransactor.Value = amount chbookRaw := &contract.ChequebookRaw{Contract: self.contract} tx, err := chbookRaw.Transfer(depositTransactor) diff --git a/contracts/randomize/randomize_test.go b/contracts/randomize/randomize_test.go index 87320dde05..ce4722bb7f 100644 --- a/contracts/randomize/randomize_test.go +++ b/contracts/randomize/randomize_test.go @@ -57,7 +57,6 @@ func TestRandomize(t *testing.T) { transactOpts.GasLimit = 1000000 randomizeAddress, randomize, err := DeployRandomize(transactOpts, contractBackend) - t.Log("contract address", randomizeAddress.String()) if err != nil { t.Fatalf("can't deploy root registry: %v", err) } @@ -66,18 +65,15 @@ func TestRandomize(t *testing.T) { d := time.Now().Add(1000 * time.Millisecond) ctx, cancel := context.WithDeadline(context.Background(), d) defer cancel() - code, _ := contractBackend.CodeAt(ctx, randomizeAddress, nil) - t.Log("contract code", common.ToHex(code)) + contractBackend.CodeAt(ctx, randomizeAddress, nil) f := func(key, val common.Hash) bool { - t.Log(key.Hex(), val.Hex()) return true } contractBackend.ForEachStorageAt(ctx, randomizeAddress, nil, f) - s, err := randomize.SetSecret(byte0) + _, err = randomize.SetSecret(byte0) if err != nil { t.Fatalf("can't set secret: %v", err) } - t.Log("tx data", s) contractBackend.Commit() } @@ -151,8 +147,7 @@ func TestSendTxRandomizeSecretAndOpening(t *testing.T) { if err != nil { t.Fatalf("Can't get secret from SC: %v", err) } - randomize, err := contracts.DecryptRandomizeFromSecretsAndOpening(secrets, opening) - t.Log("randomize", randomize) + _, err = contracts.DecryptRandomizeFromSecretsAndOpening(secrets, opening) if err != nil { t.Error("Can't decrypt secret and opening", err) } diff --git a/contracts/tests/Inherited_test.go b/contracts/tests/Inherited_test.go index 90dbf4c0c1..fd858cbf32 100644 --- a/contracts/tests/Inherited_test.go +++ b/contracts/tests/Inherited_test.go @@ -2,7 +2,6 @@ package tests import ( "context" - "fmt" "math/big" "os" "testing" @@ -38,15 +37,12 @@ func TestPriceFeed(t *testing.T) { t.Fatalf("can't create TransactOpts: %v", err) } // deploy payer swap SMC - addr, contract, err := DeployMyInherited(transactOpts, contractBackend) + _, contract, err := DeployMyInherited(transactOpts, contractBackend) if err != nil { t.Fatal("can't deploy smart contract: ", err) } - fmt.Println("addr", addr.Hex()) - tx, err := contract.Foo() + _, err = contract.Foo() if err != nil { t.Fatal("can't run function Foo() in smart contract: ", err) } - fmt.Println("tx", tx) - } diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index eba7fad779..98297014a0 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -56,9 +56,11 @@ var ( qosConfidenceCap = 10 // Number of peers above which not to modify RTT confidence qosTuningImpact = 0.25 // Impact that a new tuning target has on the previous value - maxQueuedHeaders = 32 * 1024 // [eth/62] Maximum number of headers to queue for import (DOS protection) - maxHeadersProcess = 2048 // Number of header download results to import at once into the chain - maxResultsProcess = 2048 // Number of content download results to import at once into the chain + maxQueuedHeaders = 32 * 1024 // [eth/62] Maximum number of headers to queue for import (DOS protection) + maxHeadersProcess = 2048 // Number of header download results to import at once into the chain + maxResultsProcess = 2048 // Number of content download results to import at once into the chain + fullMaxForkAncestry uint64 = params.FullImmutabilityThreshold // Maximum chain reorganisation (locally redeclared so tests can reduce it) + lightMaxForkAncestry uint64 = params.LightImmutabilityThreshold // Maximum chain reorganisation (locally redeclared so tests can reduce it) fsHeaderCheckFrequency = 100 // Verification frequency of the downloaded headers during fast sync fsHeaderSafetyNet = 2048 // Number of headers to discard in case a chain violation is detected @@ -975,22 +977,22 @@ func (d *Downloader) fetchReceipts(from uint64) error { // various callbacks to handle the slight differences between processing them. // // The instrumentation parameters: -// - errCancel: error type to return if the fetch operation is cancelled (mostly makes logging nicer) -// - deliveryCh: channel from which to retrieve downloaded data packets (merged from all concurrent peers) -// - deliver: processing callback to deliver data packets into type specific download queues (usually within `queue`) -// - wakeCh: notification channel for waking the fetcher when new tasks are available (or sync completed) -// - expire: task callback method to abort requests that took too long and return the faulty peers (traffic shaping) -// - pending: task callback for the number of requests still needing download (detect completion/non-completability) -// - inFlight: task callback for the number of in-progress requests (wait for all active downloads to finish) -// - throttle: task callback to check if the processing queue is full and activate throttling (bound memory use) -// - reserve: task callback to reserve new download tasks to a particular peer (also signals partial completions) -// - fetchHook: tester callback to notify of new tasks being initiated (allows testing the scheduling logic) -// - fetch: network callback to actually send a particular download request to a physical remote peer -// - cancel: task callback to abort an in-flight download request and allow rescheduling it (in case of lost peer) -// - capacity: network callback to retrieve the estimated type-specific bandwidth capacity of a peer (traffic shaping) -// - idle: network callback to retrieve the currently (type specific) idle peers that can be assigned tasks -// - setIdle: network callback to set a peer back to idle and update its estimated capacity (traffic shaping) -// - kind: textual label of the type being downloaded to display in log mesages +// - errCancel: error type to return if the fetch operation is cancelled (mostly makes logging nicer) +// - deliveryCh: channel from which to retrieve downloaded data packets (merged from all concurrent peers) +// - deliver: processing callback to deliver data packets into type specific download queues (usually within `queue`) +// - wakeCh: notification channel for waking the fetcher when new tasks are available (or sync completed) +// - expire: task callback method to abort requests that took too long and return the faulty peers (traffic shaping) +// - pending: task callback for the number of requests still needing download (detect completion/non-completability) +// - inFlight: task callback for the number of in-progress requests (wait for all active downloads to finish) +// - throttle: task callback to check if the processing queue is full and activate throttling (bound memory use) +// - reserve: task callback to reserve new download tasks to a particular peer (also signals partial completions) +// - fetchHook: tester callback to notify of new tasks being initiated (allows testing the scheduling logic) +// - fetch: network callback to actually send a particular download request to a physical remote peer +// - cancel: task callback to abort an in-flight download request and allow rescheduling it (in case of lost peer) +// - capacity: network callback to retrieve the estimated type-specific bandwidth capacity of a peer (traffic shaping) +// - idle: network callback to retrieve the currently (type specific) idle peers that can be assigned tasks +// - setIdle: network callback to set a peer back to idle and update its estimated capacity (traffic shaping) +// - kind: textual label of the type being downloaded to display in log mesages func (d *Downloader) fetchParts(errCancel error, deliveryCh chan dataPack, deliver func(dataPack) (int, error), wakeCh chan bool, expire func() map[string]int, pending func() int, inFlight func() bool, throttle func() bool, reserve func(*peerConnection, int) (*fetchRequest, bool, error), fetchHook func([]*types.Header), fetch func(*peerConnection, *fetchRequest) error, cancel func(*fetchRequest), capacity func(*peerConnection) int, diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index ae993b26bd..4145926173 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -25,23 +25,19 @@ import ( "testing" "time" + "github.com/tomochain/tomochain" "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/consensus/ethash" "github.com/tomochain/tomochain/core" "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/types" - "github.com/tomochain/tomochain/crypto" + "github.com/tomochain/tomochain/core/vm" "github.com/tomochain/tomochain/ethdb" "github.com/tomochain/tomochain/event" "github.com/tomochain/tomochain/params" "github.com/tomochain/tomochain/trie" ) -var ( - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - testAddress = crypto.PubkeyToAddress(testKey.PublicKey) -) - // Reduce some of the parameters to make the tester faster. func init() { MaxForkAncestry = uint64(10000) @@ -71,34 +67,37 @@ type downloadTester struct { peerMissingStates map[string]map[common.Hash]bool // State entries that fast sync should not return - lock sync.RWMutex + chain *core.BlockChain + peers map[string]*downloadTesterPeer + lock sync.RWMutex } // newTester creates a new downloader test mocker. -func newTester() *downloadTester { - testdb := rawdb.NewMemoryDatabase() - genesis := core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000)) +func newTester(t *testing.T) *downloadTester { + return newTesterWithNotification(t, nil) +} +// newTester creates a new downloader test mocker. +func newTesterWithNotification(t *testing.T, success func()) *downloadTester { + db := rawdb.NewMemoryDatabase() + t.Cleanup(func() { + db.Close() + }) + gspec := &core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + gspec.MustCommit(db) + chain, err := core.NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) + if err != nil { + panic(err) + } tester := &downloadTester{ - genesis: genesis, - peerDb: testdb, - ownHashes: []common.Hash{genesis.Hash()}, - ownHeaders: map[common.Hash]*types.Header{genesis.Hash(): genesis.Header()}, - ownBlocks: map[common.Hash]*types.Block{genesis.Hash(): genesis}, - ownReceipts: map[common.Hash]types.Receipts{genesis.Hash(): nil}, - ownChainTd: map[common.Hash]*big.Int{genesis.Hash(): genesis.Difficulty()}, - peerHashes: make(map[string][]common.Hash), - peerHeaders: make(map[string]map[common.Hash]*types.Header), - peerBlocks: make(map[string]map[common.Hash]*types.Block), - peerReceipts: make(map[string]map[common.Hash]types.Receipts), - peerChainTds: make(map[string]map[common.Hash]*big.Int), - peerMissingStates: make(map[string]map[common.Hash]bool), - } - tester.stateDb = rawdb.NewMemoryDatabase() - tester.stateDb.Put(genesis.Root().Bytes(), []byte{0x00}) - - tester.downloader = New(FullSync, tester.stateDb, new(event.TypeMux), tester, nil, tester.dropPeer) - + chain: chain, + peers: make(map[string]*downloadTesterPeer), + } + tester.downloader = New(FullSync, db, new(event.TypeMux), tester.chain, nil, tester.dropPeer) return tester } @@ -388,8 +387,22 @@ func (dl *downloadTester) Rollback(hashes []common.Hash) { } // newPeer registers a new block download source into the downloader. -func (dl *downloadTester) newPeer(id string, version int, hashes []common.Hash, headers map[common.Hash]*types.Header, blocks map[common.Hash]*types.Block, receipts map[common.Hash]types.Receipts) error { - return dl.newSlowPeer(id, version, hashes, headers, blocks, receipts, 0) +func (dl *downloadTester) newPeer(id string, version uint, blocks []*types.Block) *downloadTesterPeer { + dl.lock.Lock() + defer dl.lock.Unlock() + + peer := &downloadTesterPeer{ + dl: dl, + id: id, + chain: newTestBlockchain(blocks), + withholdHeaders: make(map[common.Hash]struct{}), + } + dl.peers[id] = peer + + if err := dl.downloader.RegisterPeer(id, int(version), peer); err != nil { + panic(err) + } + return peer } // newSlowPeer registers a new block download source into the downloader, with a @@ -461,10 +474,12 @@ func (dl *downloadTester) dropPeer(id string) { func (dl *downloadTester) Config() *params.ChainConfig { return params.TestChainConfig } type downloadTesterPeer struct { - dl *downloadTester - id string - delay time.Duration - lock sync.RWMutex + dl *downloadTester + id string + chain *core.BlockChain + delay time.Duration + lock sync.RWMutex + withholdHeaders map[common.Hash]struct{} } // setDelay is a thread safe setter for the network delay value. @@ -672,23 +687,21 @@ func TestCanonicalSynchronisation64Light(t *testing.T) { testCanonicalSynchronisation(t, 64, LightSync) } -func testCanonicalSynchronisation(t *testing.T, protocol int, mode SyncMode) { +func testCanonicalSynchronisation(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() - tester := newTester() + tester := newTester(t) defer tester.terminate() // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) + chain := testChainBase.shorten(blockCacheMaxItems - 15) + tester.newPeer("peer", protocol, chain.blocks[1:]) // Synchronise with the peer and make sure all relevant data was retrieved if err := tester.sync("peer", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, len(chain.blocks)) } // Tests that if a large batch of blocks are being downloaded, it is throttled @@ -699,16 +712,14 @@ func TestThrottling63Fast(t *testing.T) { testThrottling(t, 63, FastSync) } func TestThrottling64Full(t *testing.T) { testThrottling(t, 64, FullSync) } func TestThrottling64Fast(t *testing.T) { testThrottling(t, 64, FastSync) } -func testThrottling(t *testing.T, protocol int, mode SyncMode) { +func testThrottling(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() - tester := newTester() + tester := newTester(t) defer tester.terminate() // Create a long block chain to download and the tester - targetBlocks := 8 * blockCacheItems - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) + targetBlocks := len(testChainBase.blocks) - 1 + tester.newPeer("peer", protocol, testChainBase.blocks[1:]) // Wrap the importer to allow stepping blocked, proceed := uint32(0), make(chan struct{}) @@ -786,30 +797,29 @@ func TestForkedSync64Full(t *testing.T) { testForkedSync(t, 64, FullSync) } func TestForkedSync64Fast(t *testing.T) { testForkedSync(t, 64, FastSync) } func TestForkedSync64Light(t *testing.T) { testForkedSync(t, 64, LightSync) } -func testForkedSync(t *testing.T, protocol int, mode SyncMode) { +func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { t.Parallel() - tester := newTester() + tester := newTester(t) defer tester.terminate() // Create a long enough forked chain - common, fork := MaxHashFetch, 2*MaxHashFetch - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, true) - - tester.newPeer("fork A", protocol, hashesA, headersA, blocksA, receiptsA) - tester.newPeer("fork B", protocol, hashesB, headersB, blocksB, receiptsB) + chainA := testChainForkLightA.shorten(len(testChainBase.blocks) + 80) + chainB := testChainForkLightB.shorten(len(testChainBase.blocks) + 81) + tester.newPeer("fork A", protocol, chainA.blocks[1:]) + tester.newPeer("fork B", protocol, chainB.blocks[1:]) // Synchronise with the peer and make sure all blocks were retrieved if err := tester.sync("fork A", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, common+fork+1) + assertOwnChain(t, tester, len(chainA.blocks)) // Synchronise with the second peer and make sure that fork is pulled too if err := tester.sync("fork B", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnForkedChain(t, tester, common+1, []int{common + fork + 1, common + fork + 1}) + assertOwnChain(t, tester, len(chainB.blocks)) } // Tests that synchronising against a much shorter but much heavyer fork works @@ -821,30 +831,26 @@ func TestHeavyForkedSync64Full(t *testing.T) { testHeavyForkedSync(t, 64, FullS func TestHeavyForkedSync64Fast(t *testing.T) { testHeavyForkedSync(t, 64, FastSync) } func TestHeavyForkedSync64Light(t *testing.T) { testHeavyForkedSync(t, 64, LightSync) } -func testHeavyForkedSync(t *testing.T, protocol int, mode SyncMode) { - t.Parallel() - - tester := newTester() +func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { + tester := newTester(t) defer tester.terminate() - // Create a long enough forked chain - common, fork := MaxHashFetch, 4*MaxHashFetch - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, false) - - tester.newPeer("light", protocol, hashesA, headersA, blocksA, receiptsA) - tester.newPeer("heavy", protocol, hashesB[fork/2:], headersB, blocksB, receiptsB) + chainA := testChainForkLightA.shorten(len(testChainBase.blocks) + 80) + chainB := testChainForkHeavy.shorten(len(testChainBase.blocks) + 79) + tester.newPeer("light", protocol, chainA.blocks[1:]) + tester.newPeer("heavy", protocol, chainB.blocks[1:]) // Synchronise with the peer and make sure all blocks were retrieved if err := tester.sync("light", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, common+fork+1) + assertOwnChain(t, tester, len(chainA.blocks)) // Synchronise with the second peer and make sure that fork is pulled too if err := tester.sync("heavy", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnForkedChain(t, tester, common+1, []int{common + fork + 1, common + fork/2 + 1}) + assertOwnChain(t, tester, len(chainB.blocks)) } // Tests that chain forks are contained within a certain interval of the current @@ -857,24 +863,20 @@ func TestBoundedForkedSync64Full(t *testing.T) { testBoundedForkedSync(t, 64, F func TestBoundedForkedSync64Fast(t *testing.T) { testBoundedForkedSync(t, 64, FastSync) } func TestBoundedForkedSync64Light(t *testing.T) { testBoundedForkedSync(t, 64, LightSync) } -func testBoundedForkedSync(t *testing.T, protocol int, mode SyncMode) { - t.Parallel() - - tester := newTester() +func testBoundedForkedSync(t *testing.T, protocol uint, mode SyncMode) { + tester := newTester(t) defer tester.terminate() - // Create a long enough forked chain - common, fork := 13, int(MaxForkAncestry+17) - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, true) - - tester.newPeer("original", protocol, hashesA, headersA, blocksA, receiptsA) - tester.newPeer("rewriter", protocol, hashesB, headersB, blocksB, receiptsB) + chainA := testChainForkLightA + chainB := testChainForkLightB + tester.newPeer("original", protocol, chainA.blocks[1:]) + tester.newPeer("rewriter", protocol, chainB.blocks[1:]) // Synchronise with the peer and make sure all blocks were retrieved if err := tester.sync("original", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, common+fork+1) + assertOwnChain(t, tester, len(chainA.blocks)) // Synchronise with the second peer and ensure that the fork is rejected to being too old if err := tester.sync("rewriter", nil, mode); err != errInvalidAncestor { @@ -892,25 +894,22 @@ func TestBoundedHeavyForkedSync64Full(t *testing.T) { testBoundedHeavyForkedSyn func TestBoundedHeavyForkedSync64Fast(t *testing.T) { testBoundedHeavyForkedSync(t, 64, FastSync) } func TestBoundedHeavyForkedSync64Light(t *testing.T) { testBoundedHeavyForkedSync(t, 64, LightSync) } -func testBoundedHeavyForkedSync(t *testing.T, protocol int, mode SyncMode) { - t.Parallel() - - tester := newTester() +func testBoundedHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { + tester := newTester(t) defer tester.terminate() // Create a long enough forked chain - common, fork := 13, int(MaxForkAncestry+17) - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, false) - - tester.newPeer("original", protocol, hashesA, headersA, blocksA, receiptsA) - tester.newPeer("heavy-rewriter", protocol, hashesB[MaxForkAncestry-17:], headersB, blocksB, receiptsB) // Root the fork below the ancestor limit + chainA := testChainForkLightA + chainB := testChainForkHeavy + tester.newPeer("original", protocol, chainA.blocks[1:]) // Synchronise with the peer and make sure all blocks were retrieved if err := tester.sync("original", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, common+fork+1) + assertOwnChain(t, tester, len(chainA.blocks)) + tester.newPeer("heavy-rewriter", protocol, chainB.blocks[1:]) // Synchronise with the second peer and ensure that the fork is rejected to being too old if err := tester.sync("heavy-rewriter", nil, mode); err != errInvalidAncestor { t.Fatalf("sync failure mismatch: have %v, want %v", err, errInvalidAncestor) @@ -922,7 +921,7 @@ func testBoundedHeavyForkedSync(t *testing.T, protocol int, mode SyncMode) { func TestInactiveDownloader62(t *testing.T) { t.Parallel() - tester := newTester() + tester := newTester(t) defer tester.terminate() // Check that neither block headers nor bodies are accepted @@ -939,7 +938,7 @@ func TestInactiveDownloader62(t *testing.T) { func TestInactiveDownloader63(t *testing.T) { t.Parallel() - tester := newTester() + tester := newTester(t) defer tester.terminate() // Check that neither block headers nor bodies are accepted @@ -962,23 +961,12 @@ func TestCancel64Full(t *testing.T) { testCancel(t, 64, FullSync) } func TestCancel64Fast(t *testing.T) { testCancel(t, 64, FastSync) } func TestCancel64Light(t *testing.T) { testCancel(t, 64, LightSync) } -func testCancel(t *testing.T, protocol int, mode SyncMode) { - t.Parallel() - - tester := newTester() +func testCancel(t *testing.T, protocol uint, mode SyncMode) { + tester := newTester(t) defer tester.terminate() - // Create a small enough block chain to download and the tester - targetBlocks := blockCacheItems - 15 - if targetBlocks >= MaxHashFetch { - targetBlocks = MaxHashFetch - 15 - } - if targetBlocks >= MaxHeaderFetch { - targetBlocks = MaxHeaderFetch - 15 - } - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) + chain := testChainBase.shorten(MaxHeaderFetch) + tester.newPeer("peer", protocol, chain.blocks[1:]) // Make sure canceling works with a pristine downloader tester.downloader.Cancel() @@ -1003,25 +991,22 @@ func TestMultiSynchronisation64Full(t *testing.T) { testMultiSynchronisation(t, func TestMultiSynchronisation64Fast(t *testing.T) { testMultiSynchronisation(t, 64, FastSync) } func TestMultiSynchronisation64Light(t *testing.T) { testMultiSynchronisation(t, 64, LightSync) } -func testMultiSynchronisation(t *testing.T, protocol int, mode SyncMode) { - t.Parallel() - - tester := newTester() +func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { + tester := newTester(t) defer tester.terminate() // Create various peers with various parts of the chain targetPeers := 8 - targetBlocks := targetPeers*blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(targetPeers * 100) for i := 0; i < targetPeers; i++ { id := fmt.Sprintf("peer #%d", i) - tester.newPeer(id, protocol, hashes[i*blockCacheItems:], headers, blocks, receipts) + tester.newPeer(id, protocol, chain.shorten(len(chain.blocks) / (i + 1)).blocks[1:]) } if err := tester.sync("peer #0", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, len(chain.blocks)) } // Tests that synchronisations behave well in multi-version protocol environments @@ -1033,31 +1018,28 @@ func TestMultiProtoSynchronisation64Full(t *testing.T) { testMultiProtoSync(t, func TestMultiProtoSynchronisation64Fast(t *testing.T) { testMultiProtoSync(t, 64, FastSync) } func TestMultiProtoSynchronisation64Light(t *testing.T) { testMultiProtoSync(t, 64, LightSync) } -func testMultiProtoSync(t *testing.T, protocol int, mode SyncMode) { - t.Parallel() - - tester := newTester() +func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { + tester := newTester(t) defer tester.terminate() // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheMaxItems - 15) // Create peers of every type - tester.newPeer("peer 62", 62, hashes, headers, blocks, nil) - tester.newPeer("peer 63", 63, hashes, headers, blocks, receipts) - tester.newPeer("peer 64", 64, hashes, headers, blocks, receipts) + tester.newPeer("peer 62", 62, chain.blocks[1:]) + tester.newPeer("peer 63", 63, chain.blocks[1:]) + tester.newPeer("peer 64", 64, chain.blocks[1:]) // Synchronise with the requested peer and make sure all blocks were retrieved if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, len(chain.blocks)) // Check that no peers have been dropped off - for _, version := range []int{62, 63, 64} { + for _, version := range []int{66, 67} { peer := fmt.Sprintf("peer %d", version) - if _, ok := tester.peerHashes[peer]; !ok { + if _, ok := tester.peers[peer]; !ok { t.Errorf("%s dropped", peer) } } @@ -1072,17 +1054,13 @@ func TestEmptyShortCircuit64Full(t *testing.T) { testEmptyShortCircuit(t, 64, F func TestEmptyShortCircuit64Fast(t *testing.T) { testEmptyShortCircuit(t, 64, FastSync) } func TestEmptyShortCircuit64Light(t *testing.T) { testEmptyShortCircuit(t, 64, LightSync) } -func testEmptyShortCircuit(t *testing.T, protocol int, mode SyncMode) { - t.Parallel() - - tester := newTester() +func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { + tester := newTester(t) defer tester.terminate() // Create a block chain to download - targetBlocks := 2*blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) + chain := testChainBase + tester.newPeer("peer", protocol, chain.blocks[1:]) // Instrument the downloader to signal body requests bodiesHave, receiptsHave := int32(0), int32(0) @@ -1096,17 +1074,17 @@ func testEmptyShortCircuit(t *testing.T, protocol int, mode SyncMode) { if err := tester.sync("peer", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, len(chain.blocks)) // Validate the number of block bodies that should have been requested bodiesNeeded, receiptsNeeded := 0, 0 - for _, block := range blocks { - if mode != LightSync && block != tester.genesis && (len(block.Transactions()) > 0 || len(block.Uncles()) > 0) { + for _, block := range chain.blocks[1:] { + if mode != LightSync && (len(block.Transactions()) > 0 || len(block.Uncles()) > 0) { bodiesNeeded++ } } - for _, receipt := range receipts { - if mode == FastSync && len(receipt) > 0 { + for _, block := range chain.blocks[1:] { + if mode == FastSync && len(block.Transactions()) > 0 { receiptsNeeded++ } } @@ -1127,30 +1105,24 @@ func TestMissingHeaderAttack64Full(t *testing.T) { testMissingHeaderAttack(t, 6 func TestMissingHeaderAttack64Fast(t *testing.T) { testMissingHeaderAttack(t, 64, FastSync) } func TestMissingHeaderAttack64Light(t *testing.T) { testMissingHeaderAttack(t, 64, LightSync) } -func testMissingHeaderAttack(t *testing.T, protocol int, mode SyncMode) { - t.Parallel() - - tester := newTester() +func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { + tester := newTester(t) defer tester.terminate() - // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheMaxItems - 15) - // Attempt a full sync with an attacker feeding gapped headers - tester.newPeer("attack", protocol, hashes, headers, blocks, receipts) - missing := targetBlocks / 2 - delete(tester.peerHeaders["attack"], hashes[missing]) + attacker := tester.newPeer("attack", protocol, chain.blocks[1:]) + attacker.withholdHeaders[chain.blocks[len(chain.blocks)/2-1].Hash()] = struct{}{} if err := tester.sync("attack", nil, mode); err == nil { t.Fatalf("succeeded attacker synchronisation") } // Synchronise with the valid peer and make sure sync succeeds - tester.newPeer("valid", protocol, hashes, headers, blocks, receipts) + tester.newPeer("valid", protocol, chain.blocks[1:]) if err := tester.sync("valid", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, len(chain.blocks)) } // Tests that if requested headers are shifted (i.e. first is missing), the queue @@ -1162,31 +1134,25 @@ func TestShiftedHeaderAttack64Full(t *testing.T) { testShiftedHeaderAttack(t, 6 func TestShiftedHeaderAttack64Fast(t *testing.T) { testShiftedHeaderAttack(t, 64, FastSync) } func TestShiftedHeaderAttack64Light(t *testing.T) { testShiftedHeaderAttack(t, 64, LightSync) } -func testShiftedHeaderAttack(t *testing.T, protocol int, mode SyncMode) { - t.Parallel() - - tester := newTester() +func testShiftedHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { + tester := newTester(t) defer tester.terminate() - // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheMaxItems - 15) // Attempt a full sync with an attacker feeding shifted headers - tester.newPeer("attack", protocol, hashes, headers, blocks, receipts) - delete(tester.peerHeaders["attack"], hashes[len(hashes)-2]) - delete(tester.peerBlocks["attack"], hashes[len(hashes)-2]) - delete(tester.peerReceipts["attack"], hashes[len(hashes)-2]) + attacker := tester.newPeer("attack", protocol, chain.blocks[1:]) + attacker.withholdHeaders[chain.blocks[1].Hash()] = struct{}{} if err := tester.sync("attack", nil, mode); err == nil { t.Fatalf("succeeded attacker synchronisation") } // Synchronise with the valid peer and make sure sync succeeds - tester.newPeer("valid", protocol, hashes, headers, blocks, receipts) + tester.newPeer("valid", protocol, chain.blocks[1:]) if err := tester.sync("valid", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, len(chain.blocks)) } // Tests that upon detecting an invalid header, the recent ones are rolled back @@ -1196,87 +1162,78 @@ func TestInvalidHeaderRollback63Fast(t *testing.T) { testInvalidHeaderRollback( func TestInvalidHeaderRollback64Fast(t *testing.T) { testInvalidHeaderRollback(t, 64, FastSync) } func TestInvalidHeaderRollback64Light(t *testing.T) { testInvalidHeaderRollback(t, 64, LightSync) } -func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { - t.Parallel() - - tester := newTester() +func testInvalidHeaderRollback(t *testing.T, protocol uint, mode SyncMode) { + tester := newTester(t) defer tester.terminate() // Create a small enough block chain to download targetBlocks := 3*fsHeaderSafetyNet + 256 + fsMinFullBlocks - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(targetBlocks) // Attempt to sync with an attacker that feeds junk during the fast sync phase. // This should result in the last fsHeaderSafetyNet headers being rolled back. - tester.newPeer("fast-attack", protocol, hashes, headers, blocks, receipts) missing := fsHeaderSafetyNet + MaxHeaderFetch + 1 - delete(tester.peerHeaders["fast-attack"], hashes[len(hashes)-missing]) + + fastAttacker := tester.newPeer("fast-attack", protocol, chain.blocks[1:]) + fastAttacker.withholdHeaders[chain.blocks[missing].Hash()] = struct{}{} if err := tester.sync("fast-attack", nil, mode); err == nil { t.Fatalf("succeeded fast attacker synchronisation") } - if head := tester.CurrentHeader().Number.Int64(); int(head) > MaxHeaderFetch { + if head := tester.chain.CurrentHeader().Number.Int64(); int(head) > MaxHeaderFetch { t.Errorf("rollback head mismatch: have %v, want at most %v", head, MaxHeaderFetch) } // Attempt to sync with an attacker that feeds junk during the block import phase. // This should result in both the last fsHeaderSafetyNet number of headers being // rolled back, and also the pivot point being reverted to a non-block status. - tester.newPeer("block-attack", protocol, hashes, headers, blocks, receipts) missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1 - delete(tester.peerHeaders["fast-attack"], hashes[len(hashes)-missing]) // Make sure the fast-attacker doesn't fill in - delete(tester.peerHeaders["block-attack"], hashes[len(hashes)-missing]) + + blockAttacker := tester.newPeer("block-attack", protocol, chain.blocks[1:]) + fastAttacker.withholdHeaders[chain.blocks[missing].Hash()] = struct{}{} // Make sure the fast-attacker doesn't fill in + blockAttacker.withholdHeaders[chain.blocks[missing].Hash()] = struct{}{} if err := tester.sync("block-attack", nil, mode); err == nil { t.Fatalf("succeeded block attacker synchronisation") } - if head := tester.CurrentHeader().Number.Int64(); int(head) > 2*fsHeaderSafetyNet+MaxHeaderFetch { + if head := tester.chain.CurrentHeader().Number.Int64(); int(head) > 2*fsHeaderSafetyNet+MaxHeaderFetch { t.Errorf("rollback head mismatch: have %v, want at most %v", head, 2*fsHeaderSafetyNet+MaxHeaderFetch) } if mode == FastSync { - if head := tester.CurrentBlock().NumberU64(); head != 0 { + if head := tester.chain.CurrentBlock().Number().Uint64(); head != 0 { t.Errorf("fast sync pivot block #%d not rolled back", head) } } // Attempt to sync with an attacker that withholds promised blocks after the // fast sync pivot point. This could be a trial to leave the node with a bad // but already imported pivot block. - tester.newPeer("withhold-attack", protocol, hashes, headers, blocks, receipts) - missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1 + withholdAttacker := tester.newPeer("withhold-attack", protocol, chain.blocks[1:]) tester.downloader.syncInitHook = func(uint64, uint64) { - for i := missing; i <= len(hashes); i++ { - delete(tester.peerHeaders["withhold-attack"], hashes[len(hashes)-i]) + for i := missing; i < len(chain.blocks); i++ { + withholdAttacker.withholdHeaders[chain.blocks[i].Hash()] = struct{}{} } tester.downloader.syncInitHook = nil } - if err := tester.sync("withhold-attack", nil, mode); err == nil { t.Fatalf("succeeded withholding attacker synchronisation") } - if head := tester.CurrentHeader().Number.Int64(); int(head) > 2*fsHeaderSafetyNet+MaxHeaderFetch { + if head := tester.chain.CurrentHeader().Number.Int64(); int(head) > 2*fsHeaderSafetyNet+MaxHeaderFetch { t.Errorf("rollback head mismatch: have %v, want at most %v", head, 2*fsHeaderSafetyNet+MaxHeaderFetch) } if mode == FastSync { - if head := tester.CurrentBlock().NumberU64(); head != 0 { + if head := tester.chain.CurrentBlock().Number().Uint64(); head != 0 { t.Errorf("fast sync pivot block #%d not rolled back", head) } } - // Synchronise with the valid peer and make sure sync succeeds. Since the last - // rollback should also disable fast syncing for this process, verify that we - // did a fresh full sync. Note, we can't assert anything about the receipts - // since we won't purge the database of them, hence we can't use assertOwnChain. - tester.newPeer("valid", protocol, hashes, headers, blocks, receipts) + // Synchronise with the valid peer and make sure sync succeeds. Since the last rollback + // should also disable fast syncing for this process, verify that we did a fresh full + // sync. Note, we can't assert anything about the receipts since we won't purge the + // database of them, hence we can't use assertOwnChain. + tester.newPeer("valid", protocol, chain.blocks[1:]) if err := tester.sync("valid", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - if hs := len(tester.ownHeaders); hs != len(headers) { - t.Fatalf("synchronised headers mismatch: have %v, want %v", hs, len(headers)) - } - if mode != LightSync { - if bs := len(tester.ownBlocks); bs != len(blocks) { - t.Fatalf("synchronised blocks mismatch: have %v, want %v", bs, len(blocks)) - } - } + assertOwnChain(t, tester, len(chain.blocks)) } // Tests that a peer advertising an high TD doesn't get to stall the downloader @@ -1288,15 +1245,12 @@ func TestHighTDStarvationAttack64Full(t *testing.T) { testHighTDStarvationAttac func TestHighTDStarvationAttack64Fast(t *testing.T) { testHighTDStarvationAttack(t, 64, FastSync) } func TestHighTDStarvationAttack64Light(t *testing.T) { testHighTDStarvationAttack(t, 64, LightSync) } -func testHighTDStarvationAttack(t *testing.T, protocol int, mode SyncMode) { - t.Parallel() - - tester := newTester() +func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { + tester := newTester(t) defer tester.terminate() - hashes, headers, blocks, receipts := tester.makeChain(0, 0, tester.genesis, nil, false) - tester.newPeer("attack", protocol, []common.Hash{hashes[0]}, headers, blocks, receipts) - + chain := testChainBase.shorten(1) + tester.newPeer("attack", protocol, chain.blocks[1:]) if err := tester.sync("attack", big.NewInt(1000000), mode); err != errStallingPeer { t.Fatalf("synchronisation error mismatch: have %v, want %v", err, errStallingPeer) } @@ -1307,7 +1261,7 @@ func TestBlockHeaderAttackerDropping62(t *testing.T) { testBlockHeaderAttackerDr func TestBlockHeaderAttackerDropping63(t *testing.T) { testBlockHeaderAttackerDropping(t, 63) } func TestBlockHeaderAttackerDropping64(t *testing.T) { testBlockHeaderAttackerDropping(t, 64) } -func testBlockHeaderAttackerDropping(t *testing.T, protocol int) { +func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { t.Parallel() // Define the disconnection requirement for individual hash fetch errors @@ -1337,16 +1291,15 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol int) { {errCancelContentProcessing, false}, // Synchronisation was canceled, origin may be innocent, don't drop } // Run the tests and check disconnection status - tester := newTester() + tester := newTester(t) defer tester.terminate() + chain := testChainBase.shorten(1) for i, tt := range tests { - // Register a new peer and ensure it's presence + // Register a new peer and ensure its presence id := fmt.Sprintf("test %d", i) - if err := tester.newPeer(id, protocol, []common.Hash{tester.genesis.Hash()}, nil, nil, nil); err != nil { - t.Fatalf("test %d: failed to register new peer: %v", i, err) - } - if _, ok := tester.peerHashes[id]; !ok { + tester.newPeer(id, protocol, chain.blocks[1:]) + if _, ok := tester.peers[id]; !ok { t.Fatalf("test %d: registered peer not found", i) } // Simulate a synchronisation and check the required result @@ -1368,15 +1321,11 @@ func TestSyncProgress64Full(t *testing.T) { testSyncProgress(t, 64, FullSync) } func TestSyncProgress64Fast(t *testing.T) { testSyncProgress(t, 64, FastSync) } func TestSyncProgress64Light(t *testing.T) { testSyncProgress(t, 64, LightSync) } -func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { - t.Parallel() - - tester := newTester() +func testSyncProgress(t *testing.T, protocol uint, mode SyncMode) { + tester := newTester(t) defer tester.terminate() - // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheMaxItems - 15) // Set a sync init hook to catch progress changes starting := make(chan struct{}) @@ -1386,12 +1335,10 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { starting <- struct{}{} <-progress } - // Retrieve the sync progress and ensure they are zero (pristine sync) - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { - t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) - } + checkProgress(t, tester.downloader, "pristine", tomochain.SyncProgress{}) + // Synchronise half the blocks and check initial progress - tester.newPeer("peer-half", protocol, hashes[targetBlocks/2:], headers, blocks, receipts) + tester.newPeer("peer-half", protocol, chain.shorten(len(chain.blocks) / 2).blocks[1:]) pending := new(sync.WaitGroup) pending.Add(1) @@ -1402,16 +1349,15 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks/2+1) { - t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks/2+1) - } + checkProgress(t, tester.downloader, "initial", tomochain.SyncProgress{ + HighestBlock: uint64(len(chain.blocks)/2 - 1), + }) progress <- struct{}{} pending.Wait() // Synchronise all the blocks and check continuation progress - tester.newPeer("peer-full", protocol, hashes, headers, blocks, receipts) + tester.newPeer("peer-full", protocol, chain.blocks[1:]) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("peer-full", nil, mode); err != nil { @@ -1419,15 +1365,29 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(targetBlocks/2+1) || progress.CurrentBlock != uint64(targetBlocks/2+1) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2+1, targetBlocks/2+1, targetBlocks) - } + checkProgress(t, tester.downloader, "completing", tomochain.SyncProgress{ + StartingBlock: uint64(len(chain.blocks)/2 - 1), + CurrentBlock: uint64(len(chain.blocks)/2 - 1), + HighestBlock: uint64(len(chain.blocks) - 1), + }) + + // Check final progress after successful sync progress <- struct{}{} pending.Wait() + checkProgress(t, tester.downloader, "final", tomochain.SyncProgress{ + StartingBlock: uint64(len(chain.blocks)/2 - 1), + CurrentBlock: uint64(len(chain.blocks) - 1), + HighestBlock: uint64(len(chain.blocks) - 1), + }) +} - // Check final progress after successful sync - if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(targetBlocks/2+1) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2+1, targetBlocks, targetBlocks) +func checkProgress(t *testing.T, d *Downloader, stage string, want tomochain.SyncProgress) { + // Mark this method as a helper to report errors at callsite, not in here + t.Helper() + + p := d.Progress() + if p.StartingBlock != want.StartingBlock || p.CurrentBlock != want.CurrentBlock || p.HighestBlock != want.HighestBlock { + t.Fatalf("%s progress mismatch:\nhave %+v\nwant %+v", stage, p, want) } } @@ -1441,15 +1401,12 @@ func TestForkedSyncProgress64Full(t *testing.T) { testForkedSyncProgress(t, 64, func TestForkedSyncProgress64Fast(t *testing.T) { testForkedSyncProgress(t, 64, FastSync) } func TestForkedSyncProgress64Light(t *testing.T) { testForkedSyncProgress(t, 64, LightSync) } -func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { - t.Parallel() - - tester := newTester() +func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { + tester := newTester(t) defer tester.terminate() - // Create a forked chain to simulate origin revertal - common, fork := MaxHashFetch, 2*MaxHashFetch - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, true) + chainA := testChainForkLightA.shorten(len(testChainBase.blocks) + MaxHeaderFetch) + chainB := testChainForkLightB.shorten(len(testChainBase.blocks) + MaxHeaderFetch) // Set a sync init hook to catch progress changes starting := make(chan struct{}) @@ -1459,15 +1416,12 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { starting <- struct{}{} <-progress } - // Retrieve the sync progress and ensure they are zero (pristine sync) - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { - t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) - } + checkProgress(t, tester.downloader, "pristine", tomochain.SyncProgress{}) + // Synchronise with one of the forks and check progress - tester.newPeer("fork A", protocol, hashesA, headersA, blocksA, receiptsA) + tester.newPeer("fork A", protocol, chainA.blocks[1:]) pending := new(sync.WaitGroup) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("fork A", nil, mode); err != nil { @@ -1475,9 +1429,10 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(len(hashesA)-1) { - t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, len(hashesA)-1) - } + + checkProgress(t, tester.downloader, "initial", tomochain.SyncProgress{ + HighestBlock: uint64(len(chainA.blocks) - 1), + }) progress <- struct{}{} pending.Wait() @@ -1485,9 +1440,8 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { tester.downloader.syncStatsChainOrigin = tester.downloader.syncStatsChainHeight // Synchronise with the second fork and check progress resets - tester.newPeer("fork B", protocol, hashesB, headersB, blocksB, receiptsB) + tester.newPeer("fork B", protocol, chainB.blocks[1:]) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("fork B", nil, mode); err != nil { @@ -1495,16 +1449,20 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(common) || progress.CurrentBlock != uint64(len(hashesA)-1) || progress.HighestBlock != uint64(len(hashesB)-1) { - t.Fatalf("Forking progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, common, len(hashesA)-1, len(hashesB)-1) - } - progress <- struct{}{} - pending.Wait() + checkProgress(t, tester.downloader, "forking", tomochain.SyncProgress{ + StartingBlock: uint64(len(testChainBase.blocks)) - 1, + CurrentBlock: uint64(len(chainA.blocks) - 1), + HighestBlock: uint64(len(chainB.blocks) - 1), + }) // Check final progress after successful sync - if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(common) || progress.CurrentBlock != uint64(len(hashesB)-1) || progress.HighestBlock != uint64(len(hashesB)-1) { - t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, common, len(hashesB)-1, len(hashesB)-1) - } + progress <- struct{}{} + pending.Wait() + checkProgress(t, tester.downloader, "final", tomochain.SyncProgress{ + StartingBlock: uint64(len(testChainBase.blocks)) - 1, + CurrentBlock: uint64(len(chainB.blocks) - 1), + HighestBlock: uint64(len(chainB.blocks) - 1), + }) } // Tests that if synchronisation is aborted due to some failure, then the progress @@ -1517,15 +1475,11 @@ func TestFailedSyncProgress64Full(t *testing.T) { testFailedSyncProgress(t, 64, func TestFailedSyncProgress64Fast(t *testing.T) { testFailedSyncProgress(t, 64, FastSync) } func TestFailedSyncProgress64Light(t *testing.T) { testFailedSyncProgress(t, 64, LightSync) } -func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { - t.Parallel() - - tester := newTester() +func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { + tester := newTester(t) defer tester.terminate() - // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheMaxItems - 15) // Set a sync init hook to catch progress changes starting := make(chan struct{}) @@ -1535,20 +1489,16 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { starting <- struct{}{} <-progress } - // Retrieve the sync progress and ensure they are zero (pristine sync) - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { - t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) - } + checkProgress(t, tester.downloader, "pristine", tomochain.SyncProgress{}) + // Attempt a full sync with a faulty peer - tester.newPeer("faulty", protocol, hashes, headers, blocks, receipts) - missing := targetBlocks / 2 - delete(tester.peerHeaders["faulty"], hashes[missing]) - delete(tester.peerBlocks["faulty"], hashes[missing]) - delete(tester.peerReceipts["faulty"], hashes[missing]) + missing := len(chain.blocks)/2 - 1 + + faulter := tester.newPeer("faulty", protocol, chain.blocks[1:]) + faulter.withholdHeaders[chain.blocks[missing].Hash()] = struct{}{} pending := new(sync.WaitGroup) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("faulty", nil, mode); err == nil { @@ -1556,16 +1506,17 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks) - } + checkProgress(t, tester.downloader, "initial", tomochain.SyncProgress{ + HighestBlock: uint64(len(chain.blocks) - 1), + }) progress <- struct{}{} pending.Wait() + afterFailedSync := tester.downloader.Progress() - // Synchronise with a good peer and check that the progress origin remind the same after a failure - tester.newPeer("valid", protocol, hashes, headers, blocks, receipts) + // Synchronise with a good peer and check that the progress origin remind the same + // after a failure + tester.newPeer("valid", protocol, chain.blocks[1:]) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("valid", nil, mode); err != nil { @@ -1573,16 +1524,15 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock > uint64(targetBlocks/2) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/0-%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, targetBlocks/2, targetBlocks) - } - progress <- struct{}{} - pending.Wait() + checkProgress(t, tester.downloader, "completing", afterFailedSync) // Check final progress after successful sync - if progress := tester.downloader.Progress(); progress.StartingBlock > uint64(targetBlocks/2) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Final progress mismatch: have %v/%v/%v, want 0-%v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2, targetBlocks, targetBlocks) - } + progress <- struct{}{} + pending.Wait() + checkProgress(t, tester.downloader, "final", tomochain.SyncProgress{ + CurrentBlock: uint64(len(chain.blocks) - 1), + HighestBlock: uint64(len(chain.blocks) - 1), + }) } // Tests that if an attacker fakes a chain height, after the attack is detected, @@ -1594,39 +1544,29 @@ func TestFakedSyncProgress64Full(t *testing.T) { testFakedSyncProgress(t, 64, F func TestFakedSyncProgress64Fast(t *testing.T) { testFakedSyncProgress(t, 64, FastSync) } func TestFakedSyncProgress64Light(t *testing.T) { testFakedSyncProgress(t, 64, LightSync) } -func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) { - t.Parallel() - - tester := newTester() +func testFakedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { + tester := newTester(t) defer tester.terminate() - // Create a small block chain - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks+3, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheMaxItems - 15) // Set a sync init hook to catch progress changes starting := make(chan struct{}) progress := make(chan struct{}) - tester.downloader.syncInitHook = func(origin, latest uint64) { starting <- struct{}{} <-progress } - // Retrieve the sync progress and ensure they are zero (pristine sync) - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { - t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) - } - // Create and sync with an attacker that promises a higher chain than available - tester.newPeer("attack", protocol, hashes, headers, blocks, receipts) - for i := 1; i < 3; i++ { - delete(tester.peerHeaders["attack"], hashes[i]) - delete(tester.peerBlocks["attack"], hashes[i]) - delete(tester.peerReceipts["attack"], hashes[i]) - } + checkProgress(t, tester.downloader, "pristine", tomochain.SyncProgress{}) + // Create and sync with an attacker that promises a higher chain than available. + attacker := tester.newPeer("attack", protocol, chain.blocks[1:]) + numMissing := 5 + for i := len(chain.blocks) - 2; i > len(chain.blocks)-numMissing; i-- { + attacker.withholdHeaders[chain.blocks[i].Hash()] = struct{}{} + } pending := new(sync.WaitGroup) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("attack", nil, mode); err == nil { @@ -1634,14 +1574,17 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks+3) { - t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks+3) - } + checkProgress(t, tester.downloader, "initial", tomochain.SyncProgress{ + HighestBlock: uint64(len(chain.blocks) - 1), + }) progress <- struct{}{} pending.Wait() + afterFailedSync := tester.downloader.Progress() - // Synchronise with a good peer and check that the progress height has been reduced to the true value - tester.newPeer("valid", protocol, hashes[3:], headers, blocks, receipts) + // Synchronise with a good peer and check that the progress height has been reduced to + // the true value. + validChain := chain.shorten(len(chain.blocks) - numMissing) + tester.newPeer("valid", protocol, validChain.blocks[1:]) pending.Add(1) go func() { @@ -1651,16 +1594,17 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock > uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/0-%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, targetBlocks, targetBlocks) - } + checkProgress(t, tester.downloader, "completing", tomochain.SyncProgress{ + CurrentBlock: afterFailedSync.CurrentBlock, + HighestBlock: uint64(len(validChain.blocks) - 1), + }) + // Check final progress after successful sync. progress <- struct{}{} pending.Wait() - - // Check final progress after successful sync - if progress := tester.downloader.Progress(); progress.StartingBlock > uint64(targetBlocks) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Final progress mismatch: have %v/%v/%v, want 0-%v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks, targetBlocks, targetBlocks) - } + checkProgress(t, tester.downloader, "final", tomochain.SyncProgress{ + CurrentBlock: uint64(len(validChain.blocks) - 1), + HighestBlock: uint64(len(validChain.blocks) - 1), + }) } // This test reproduces an issue where unexpected deliveries would @@ -1669,7 +1613,7 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) { // and not with the other tests, avoiding intermittent failures. func TestDeliverHeadersHang(t *testing.T) { testCases := []struct { - protocol int + protocol uint syncMode SyncMode }{ {62, FullSync}, @@ -1686,6 +1630,30 @@ func TestDeliverHeadersHang(t *testing.T) { } } +func testDeliverHeadersHang(t *testing.T, protocol uint, mode SyncMode) { + master := newTester(t) + defer master.terminate() + + chain := testChainBase.shorten(15) + + for i := 0; i < 200; i++ { + tester := newTester(t) + tester.peerDb = master.peerDb + tester.newPeer("peer", protocol, chain.blocks[1:]) + + // Whenever the downloader requests headers, flood it with + // a lot of unrequested header deliveries. + tester.downloader.peers.peers["peer"].peer = &floodingTestPeer{ + peer: tester.downloader.peers.peers["peer"].peer, + tester: tester, + } + if err := tester.sync("peer", nil, mode); err != nil { + t.Errorf("test %d: sync failed: %v", i, err) + } + tester.terminate() + } +} + type floodingTestPeer struct { peer Peer tester *downloadTester @@ -1731,31 +1699,3 @@ func (ftp *floodingTestPeer) RequestHeadersByNumber(from uint64, count, skip int } return nil } - -func testDeliverHeadersHang(t *testing.T, protocol int, mode SyncMode) { - t.Parallel() - - master := newTester() - defer master.terminate() - - hashes, headers, blocks, receipts := master.makeChain(5, 0, master.genesis, nil, false) - for i := 0; i < 200; i++ { - tester := newTester() - tester.peerDb = master.peerDb - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) - // Whenever the downloader requests headers, flood it with - // a lot of unrequested header deliveries. - tester.downloader.peers.peers["peer"].peer = &floodingTestPeer{ - peer: tester.downloader.peers.peers["peer"].peer, - tester: tester, - } - if err := tester.sync("peer", nil, mode); err != nil { - t.Errorf("test %d: sync failed: %v", i, err) - } - tester.terminate() - - // Flush all goroutines to prevent messing with subsequent tests - tester.downloader.peers.peers["peer"].peer.(*floodingTestPeer).pend.Wait() - } -} diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index 0ed4e75faa..3dd9f81319 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -33,6 +33,7 @@ import ( ) var ( + blockCacheMaxItems = 8192 // Maximum number of blocks to cache before throttling the download blockCacheItems = 8192 // Maximum number of blocks to cache before throttling the download blockCacheMemory = 64 * 1024 * 1024 // Maximum amount of memory to use for block caching blockCacheSizeWeight = 0.1 // Multiplier to approximate the average block size based on past ones diff --git a/eth/downloader/testchain_test.go b/eth/downloader/testchain_test.go new file mode 100644 index 0000000000..debdd56b2e --- /dev/null +++ b/eth/downloader/testchain_test.go @@ -0,0 +1,230 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package downloader + +import ( + "fmt" + "math/big" + "sync" + "time" + + "github.com/tomochain/tomochain/common" + "github.com/tomochain/tomochain/consensus/ethash" + "github.com/tomochain/tomochain/core" + "github.com/tomochain/tomochain/core/rawdb" + "github.com/tomochain/tomochain/core/types" + "github.com/tomochain/tomochain/core/vm" + "github.com/tomochain/tomochain/crypto" + "github.com/tomochain/tomochain/params" +) + +// Test chain parameters. +var ( + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + testAddress = crypto.PubkeyToAddress(testKey.PublicKey) + testDB = rawdb.NewMemoryDatabase() + + testGspec = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + testGenesis = testGspec.MustCommit(testDB) +) + +// The common prefix of all test chains: +var testChainBase *testChain + +// Different forks on top of the base chain: +var testChainForkLightA, testChainForkLightB, testChainForkHeavy *testChain + +var pregenerated bool + +func init() { + // Reduce some of the parameters to make the tester faster + fullMaxForkAncestry = 10000 + lightMaxForkAncestry = 10000 + blockCacheMaxItems = 1024 + fsHeaderSafetyNet = 256 + fsHeaderContCheck = 500 * time.Millisecond + + testChainBase = newTestChain(blockCacheMaxItems+200, testGenesis) + + var forkLen = int(fullMaxForkAncestry + 50) + var wg sync.WaitGroup + + // Generate the test chains to seed the peers with + wg.Add(3) + go func() { testChainForkLightA = testChainBase.makeFork(forkLen, false, 1); wg.Done() }() + go func() { testChainForkLightB = testChainBase.makeFork(forkLen, false, 2); wg.Done() }() + go func() { testChainForkHeavy = testChainBase.makeFork(forkLen, true, 3); wg.Done() }() + wg.Wait() + + // Generate the test peers used by the tests to avoid overloading during testing. + // These seemingly random chains are used in various downloader tests. We're just + // pre-generating them here. + chains := []*testChain{ + testChainBase, + testChainForkLightA, + testChainForkLightB, + testChainForkHeavy, + testChainBase.shorten(1), + testChainBase.shorten(blockCacheMaxItems - 15), + testChainBase.shorten((blockCacheMaxItems - 15) / 2), + testChainBase.shorten(blockCacheMaxItems - 15 - 5), + testChainBase.shorten(MaxHeaderFetch), + testChainBase.shorten(800), + testChainBase.shorten(800 / 2), + testChainBase.shorten(800 / 3), + testChainBase.shorten(800 / 4), + testChainBase.shorten(800 / 5), + testChainBase.shorten(800 / 6), + testChainBase.shorten(800 / 7), + testChainBase.shorten(800 / 8), + testChainBase.shorten(3*fsHeaderSafetyNet + 256 + fsMinFullBlocks), + testChainBase.shorten(fsMinFullBlocks + 256 - 1), + testChainForkLightA.shorten(len(testChainBase.blocks) + 80), + testChainForkLightB.shorten(len(testChainBase.blocks) + 81), + testChainForkLightA.shorten(len(testChainBase.blocks) + MaxHeaderFetch), + testChainForkLightB.shorten(len(testChainBase.blocks) + MaxHeaderFetch), + testChainForkHeavy.shorten(len(testChainBase.blocks) + 79), + } + wg.Add(len(chains)) + for _, chain := range chains { + go func(blocks []*types.Block) { + newTestBlockchain(blocks) + wg.Done() + }(chain.blocks[1:]) + } + wg.Wait() + + // Mark the chains pregenerated. Generating a new one will lead to a panic. + pregenerated = true +} + +type testChain struct { + blocks []*types.Block +} + +// newTestChain creates a blockchain of the given length. +func newTestChain(length int, genesis *types.Block) *testChain { + tc := &testChain{ + blocks: []*types.Block{genesis}, + } + tc.generate(length-1, 0, genesis, false) + return tc +} + +// makeFork creates a fork on top of the test chain. +func (tc *testChain) makeFork(length int, heavy bool, seed byte) *testChain { + fork := tc.copy(len(tc.blocks) + length) + fork.generate(length, seed, tc.blocks[len(tc.blocks)-1], heavy) + return fork +} + +// shorten creates a copy of the chain with the given length. It panics if the +// length is longer than the number of available blocks. +func (tc *testChain) shorten(length int) *testChain { + if length > len(tc.blocks) { + panic(fmt.Errorf("can't shorten test chain to %d blocks, it's only %d blocks long", length, len(tc.blocks))) + } + return tc.copy(length) +} + +func (tc *testChain) copy(newlen int) *testChain { + if newlen > len(tc.blocks) { + newlen = len(tc.blocks) + } + cpy := &testChain{ + blocks: append([]*types.Block{}, tc.blocks[:newlen]...), + } + return cpy +} + +// generate creates a chain of n blocks starting at and including parent. +// the returned hash chain is ordered head->parent. In addition, every 22th block +// contains a transaction and every 5th an uncle to allow testing correct block +// reassembly. +func (tc *testChain) generate(n int, seed byte, parent *types.Block, heavy bool) { + blocks, _ := core.GenerateChain(testGspec.Config, parent, ethash.NewFaker(), testDB, n, func(i int, block *core.BlockGen) { + block.SetCoinbase(common.Address{seed}) + // If a heavy chain is requested, delay blocks to raise difficulty + if heavy { + block.OffsetTime(-9) + } + // Include transactions to the miner to make blocks more interesting. + if parent == tc.blocks[0] && i%22 == 0 { + signer := types.MakeSigner(params.TestChainConfig, block.Number()) + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, block.BaseFee(), nil), signer, testKey) + if err != nil { + panic(err) + } + block.AddTx(tx) + } + // if the block number is a multiple of 5, add a bonus uncle to the block + if i > 0 && i%5 == 0 { + block.AddUncle(&types.Header{ + ParentHash: block.PrevBlock(i - 2).Hash(), + Number: big.NewInt(block.Number().Int64() - 1), + }) + } + }) + tc.blocks = append(tc.blocks, blocks...) +} + +var ( + testBlockchains = make(map[common.Hash]*testBlockchain) + testBlockchainsLock sync.Mutex +) + +type testBlockchain struct { + chain *core.BlockChain + gen sync.Once +} + +// newTestBlockchain creates a blockchain database built by running the given blocks, +// either actually running them, or reusing a previously created one. The returned +// chains are *shared*, so *do not* mutate them. +func newTestBlockchain(blocks []*types.Block) *core.BlockChain { + // Retrieve an existing database, or create a new one + head := testGenesis.Hash() + if len(blocks) > 0 { + head = blocks[len(blocks)-1].Hash() + } + testBlockchainsLock.Lock() + if _, ok := testBlockchains[head]; !ok { + testBlockchains[head] = new(testBlockchain) + } + tbc := testBlockchains[head] + testBlockchainsLock.Unlock() + + // Ensure that the database is generated + tbc.gen.Do(func() { + if pregenerated { + panic("Requested chain generation outside of init") + } + chain, err := core.NewBlockChain(testDB, nil, testGspec.Config, ethash.NewFaker(), vm.Config{}) + if err != nil { + panic(err) + } + if n, err := chain.InsertChain(blocks); err != nil { + panic(fmt.Sprintf("block %d: %v", n, err)) + } + tbc.chain = chain + }) + return tbc.chain +} diff --git a/params/network_params.go b/params/network_params.go index 536a46d3df..6a0f4a2248 100644 --- a/params/network_params.go +++ b/params/network_params.go @@ -23,4 +23,16 @@ const ( // BloomBitsBlocks is the number of blocks a single bloom bit section vector // contains. BloomBitsBlocks uint64 = 4096 + + // FullImmutabilityThreshold is the number of blocks after which a chain segment is + // considered immutable (i.e. soft finality). It is used by the downloader as a + // hard limit against deep ancestors, by the blockchain against deep reorgs, by + // the freezer as the cutoff threshold and by clique as the snapshot trust limit. + FullImmutabilityThreshold = 90000 + + // LightImmutabilityThreshold is the number of blocks after which a header chain + // segment is considered immutable for light client(i.e. soft finality). It is used by + // the downloader as a hard limit against deep ancestors, by the blockchain against deep + // reorgs, by the light pruner as the pruning validity guarantee. + LightImmutabilityThreshold = 30000 ) From 25a4ac270d7fc836b969abff02d5a42f5a588be2 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Sun, 9 Jul 2023 23:13:50 +0700 Subject: [PATCH 66/82] [WIP] Fix puppeth --- accounts/abi/bind/backends/simulated.go | 9 +++++++++ cmd/puppeth/wizard_genesis.go | 9 +++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 8fb4f0f5c4..8ad9f592fe 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -71,6 +71,12 @@ type SimulatedBackend struct { // NewSimulatedBackend creates a new binding backend using a simulated blockchain // for testing purposes. func NewSimulatedBackend(alloc core.GenesisAlloc) *SimulatedBackend { + return NewSimulatedBackendWithChainConfig(alloc, nil) +} + +// NewSimulatedBackendWithChainConfig creates a new binding backend using a simulated blockchain +// with custom chain config for testing purposes. +func NewSimulatedBackendWithChainConfig(alloc core.GenesisAlloc, config *params.ChainConfig) *SimulatedBackend { database := rawdb.NewMemoryDatabase() genesis := core.Genesis{ Config: params.AllEthashProtocolChanges, @@ -78,6 +84,9 @@ func NewSimulatedBackend(alloc core.GenesisAlloc) *SimulatedBackend { GasLimit: 42000000, BaseFee: big.NewInt(params.InitialBaseFee), } + if config != nil { + genesis.Config = config + } genesis.MustCommit(database) blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}) diff --git a/cmd/puppeth/wizard_genesis.go b/cmd/puppeth/wizard_genesis.go index 6b8baf071c..ecd32f415d 100644 --- a/cmd/puppeth/wizard_genesis.go +++ b/cmd/puppeth/wizard_genesis.go @@ -56,7 +56,8 @@ func (w *wizard) makeGenesis() { EIP155Block: big.NewInt(3), EIP158Block: big.NewInt(3), ByzantiumBlock: big.NewInt(4), - LondonBlock: big.NewInt(50), + // TODO(trinhdn2): test precompiled contracts pre-London and post-London + LondonBlock: big.NewInt(0), }, } // Figure out which consensus engine to choose @@ -124,6 +125,10 @@ func (w *wizard) makeGenesis() { Epoch: 30000, Reward: 0, } + fmt.Println() + fmt.Println("Specify your chain/network ID if you want an explicit one (default = random)") + genesis.Config.ChainId = new(big.Int).SetUint64(uint64(w.readDefaultInt(rand.Intn(65536)))) + fmt.Println() fmt.Println("How many seconds should blocks take? (default = 2)") genesis.Config.Posv.Period = uint64(w.readDefaultInt(2)) @@ -184,7 +189,7 @@ func (w *wizard) makeGenesis() { // Validator Smart Contract Code pKey, _ := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr := crypto.PubkeyToAddress(pKey.PublicKey) - contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(1000000000)}}) + contractBackend := backends.NewSimulatedBackendWithChainConfig(core.GenesisAlloc{addr: {Balance: big.NewInt(1000000000)}}, genesis.Config) transactOpts, err := bind.NewKeyedTransactorWithChainID(pKey, genesis.Config.ChainId) if err != nil { fmt.Println("Can't create TransactionOpts:", err) From 4a5b98cfe6888f4bc1365e82aedf942bf3ca9d1f Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Mon, 10 Jul 2023 14:29:01 +0700 Subject: [PATCH 67/82] Fix puppeth --- accounts/abi/bind/backends/simulated.go | 27 +++++++++++++++++++------ cmd/puppeth/wizard_genesis.go | 12 ++++++----- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 8ad9f592fe..4a9b287ae2 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -431,7 +431,7 @@ func (b *SimulatedBackend) EstimateGas(ctx context.Context, call tomochain.CallM cap = hi // Create a helper to check if a gas allowance results in an executable transaction - executable := func(gas uint64) bool { + executable := func(gas uint64) (bool, error) { call.Gas = gas snapshot := b.pendingState.Snapshot() @@ -439,14 +439,21 @@ func (b *SimulatedBackend) EstimateGas(ctx context.Context, call tomochain.CallM b.pendingState.RevertToSnapshot(snapshot) if err != nil || failed { - return false + return true, err } - return true + return false, nil } // Execute the binary search and hone in on an executable gas limit for lo+1 < hi { mid := (hi + lo) / 2 - if !executable(mid) { + failed, err := executable(mid) + // If the error is not nil(consensus error), it means the provided message + // call or transaction will never be accepted no matter how much gas it is + // assigned. Return the error directly, don't struggle anymore + if err != nil { + return 0, err + } + if failed { lo = mid } else { hi = mid @@ -454,8 +461,16 @@ func (b *SimulatedBackend) EstimateGas(ctx context.Context, call tomochain.CallM } // Reject the transaction as invalid if it still fails at the highest allowance if hi == cap { - if !executable(hi) { - return 0, errGasEstimationFailed + failed, err := executable(hi) + if err != nil { + return 0, err + } + if failed { + if err != vm.ErrOutOfGas { + return 0, err + } + // Otherwise, the specified gas cap is too low + return 0, fmt.Errorf("gas required exceeds allowance (%d)", cap) } } return hi, nil diff --git a/cmd/puppeth/wizard_genesis.go b/cmd/puppeth/wizard_genesis.go index ecd32f415d..fec45f6c79 100644 --- a/cmd/puppeth/wizard_genesis.go +++ b/cmd/puppeth/wizard_genesis.go @@ -57,7 +57,7 @@ func (w *wizard) makeGenesis() { EIP158Block: big.NewInt(3), ByzantiumBlock: big.NewInt(4), // TODO(trinhdn2): test precompiled contracts pre-London and post-London - LondonBlock: big.NewInt(0), + LondonBlock: big.NewInt(1), }, } // Figure out which consensus engine to choose @@ -125,9 +125,7 @@ func (w *wizard) makeGenesis() { Epoch: 30000, Reward: 0, } - fmt.Println() - fmt.Println("Specify your chain/network ID if you want an explicit one (default = random)") - genesis.Config.ChainId = new(big.Int).SetUint64(uint64(w.readDefaultInt(rand.Intn(65536)))) + genesis.Config.ChainId = params.AllEthashProtocolChanges.ChainId fmt.Println() fmt.Println("How many seconds should blocks take? (default = 2)") @@ -189,7 +187,7 @@ func (w *wizard) makeGenesis() { // Validator Smart Contract Code pKey, _ := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr := crypto.PubkeyToAddress(pKey.PublicKey) - contractBackend := backends.NewSimulatedBackendWithChainConfig(core.GenesisAlloc{addr: {Balance: big.NewInt(1000000000)}}, genesis.Config) + contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(1_000_000_000_000_000_000)}}) transactOpts, err := bind.NewKeyedTransactorWithChainID(pKey, genesis.Config.ChainId) if err != nil { fmt.Println("Can't create TransactionOpts:", err) @@ -357,6 +355,10 @@ func (w *wizard) makeGenesis() { genesis.Alloc[common.BigToAddress(big.NewInt(i))] = core.GenesisAccount{Balance: big.NewInt(0)} } + fmt.Println() + fmt.Println("Specify your chain/network ID if you want an explicit one (default = random)") + genesis.Config.ChainId = new(big.Int).SetUint64(uint64(w.readDefaultInt(rand.Intn(65536)))) + // All done, store the genesis and flush to disk log.Info("Configured new genesis block") From 76627c8b8bea0bd588d8a1118be7de73e05e9ae0 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Tue, 11 Jul 2023 23:33:04 +0700 Subject: [PATCH 68/82] [WIP] Fix contracts unit tests --- accounts/abi/bind/backends/simulated.go | 13 ++++++++----- contracts/utils.go | 2 +- contracts/utils_test.go | 25 +++++++++++++++++++------ core/blockchain.go | 5 +++++ core/state_transition.go | 2 +- 5 files changed, 34 insertions(+), 13 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 4a9b287ae2..4d9642d9f8 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -133,7 +133,7 @@ func (b *SimulatedBackend) rollback(parent *types.Block) { blocks, _ := core.GenerateChain(b.config, parent, ethash.NewFaker(), b.database, 1, func(int, *core.BlockGen) {}) b.pendingBlock = blocks[0] - b.pendingState, _ = state.New(b.pendingBlock.Root(), state.NewDatabase(b.database)) + b.pendingState, _ = state.New(b.pendingBlock.Root(), b.blockchain.StateCache()) } // CodeAt returns the code associated with a certain account in the blockchain. @@ -438,10 +438,13 @@ func (b *SimulatedBackend) EstimateGas(ctx context.Context, call tomochain.CallM _, _, failed, err := b.callContract(ctx, call, b.pendingBlock, b.pendingState) b.pendingState.RevertToSnapshot(snapshot) - if err != nil || failed { - return true, err + if err != nil { + if errors.Is(err, core.ErrIntrinsicGas) || errors.Is(err, vm.ErrCodeStoreOutOfGas) || errors.Is(err, vm.ErrOutOfGas) { + return true, nil // Special cases, raise gas limit + } + return true, err // Bail out } - return false, nil + return failed, err } // Execute the binary search and hone in on an executable gas limit for lo+1 < hi { @@ -530,7 +533,7 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call tomochain.Call GasTipCap: call.GasTipCap, Data: call.Data, AccessList: call.AccessList, - SkipAccountChecks: false, + SkipAccountChecks: true, } feeCapacity := state.GetTRC21FeeCapacityFromState(statedb) if msg.To != nil { diff --git a/contracts/utils.go b/contracts/utils.go index 9f75c9779d..0425f949ed 100644 --- a/contracts/utils.go +++ b/contracts/utils.go @@ -174,7 +174,7 @@ func CreateTxSign(blockNumber *big.Int, blockHash common.Hash, nonce uint64, blo To: &blockSigner, Value: big.NewInt(0), Gas: 200000, - GasPrice: big.NewInt(0), + GasPrice: big.NewInt(params.InitialBaseFee), Data: inputData, }) diff --git a/contracts/utils_test.go b/contracts/utils_test.go index 016684600f..c72afca708 100644 --- a/contracts/utils_test.go +++ b/contracts/utils_test.go @@ -19,6 +19,11 @@ import ( "bytes" "context" "crypto/ecdsa" + "math/big" + "math/rand" + "testing" + "time" + "github.com/tomochain/tomochain/accounts/abi/bind" "github.com/tomochain/tomochain/accounts/abi/bind/backends" "github.com/tomochain/tomochain/common" @@ -27,10 +32,6 @@ import ( "github.com/tomochain/tomochain/core" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/crypto" - "math/big" - "math/rand" - "testing" - "time" ) var ( @@ -45,7 +46,12 @@ var ( ) func getCommonBackend() *backends.SimulatedBackend { - genesis := core.GenesisAlloc{acc1Addr: {Balance: big.NewInt(1000000000000)}} + genesis := core.GenesisAlloc{ + acc1Addr: {Balance: big.NewInt(10_000_000_000_000_000)}, + acc2Addr: {Balance: big.NewInt(10_000_000_000_000_000)}, + acc3Addr: {Balance: big.NewInt(10_000_000_000_000_000)}, + acc4Addr: {Balance: big.NewInt(10_000_000_000_000_000)}, + } backend := backends.NewSimulatedBackend(genesis) backend.Commit() @@ -59,7 +65,14 @@ func TestSendTxSign(t *testing.T) { signer := types.HomesteadSigner{} ctx := context.Background() - transactOpts := bind.NewKeyedTransactor(acc1Key) + chainID, err := backend.ChainID(context.Background()) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + transactOpts, err := bind.NewKeyedTransactorWithChainID(acc1Key, chainID) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } blockSignerAddr, blockSigner, err := blocksigner.DeployBlockSigner(transactOpts, backend, big.NewInt(99)) if err != nil { t.Fatalf("Can't get block signer: %v", err) diff --git a/core/blockchain.go b/core/blockchain.go index 4a86dcbc41..1186ea2ac3 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -495,6 +495,11 @@ func (bc *BlockChain) Processor() Processor { return bc.processor } +// StateCache returns the caching database underpinning the blockchain instance. +func (bc *BlockChain) StateCache() state.Database { + return bc.stateCache +} + // State returns a new mutable state based on the current HEAD block. func (bc *BlockChain) State() (*state.StateDB, error) { return bc.StateAt(bc.CurrentBlock().Root()) diff --git a/core/state_transition.go b/core/state_transition.go index 211b16a51f..f6e687b74c 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -364,7 +364,7 @@ func (st *StateTransition) TransitionDb(owner common.Address) (ret []byte, usedG st.state.AddBalance(st.evm.Coinbase, new(big.Int).Mul(new(big.Int).SetUint64(st.gasUsed()), st.gasPrice)) } - return ret, st.gasUsed(), vmerr != nil, err + return ret, st.gasUsed(), vmerr != nil, vmerr } func (st *StateTransition) refundGas() { From 4aa9718a3025141657492cd6872314cfd5f503c4 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 12 Jul 2023 00:00:16 +0700 Subject: [PATCH 69/82] Fix special contracts unit tests --- accounts/abi/bind/backends/simulated.go | 2 +- contracts/trc21issuer/trc21issuer_test.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 4d9642d9f8..de87a6dda1 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -444,7 +444,7 @@ func (b *SimulatedBackend) EstimateGas(ctx context.Context, call tomochain.CallM } return true, err // Bail out } - return failed, err + return false, err } // Execute the binary search and hone in on an executable gas limit for lo+1 < hi { diff --git a/contracts/trc21issuer/trc21issuer_test.go b/contracts/trc21issuer/trc21issuer_test.go index b808279a63..8206ac6ab8 100644 --- a/contracts/trc21issuer/trc21issuer_test.go +++ b/contracts/trc21issuer/trc21issuer_test.go @@ -32,7 +32,8 @@ func TestFeeTxWithTRC21Token(t *testing.T) { // init genesis contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{ - mainAddr: {Balance: big.NewInt(0).Mul(big.NewInt(10000000000000), big.NewInt(10000000000000))}, + mainAddr: {Balance: big.NewInt(0).Mul(big.NewInt(10000000000000), big.NewInt(10000000000000))}, + airdropAddr: {Balance: big.NewInt(0).Mul(big.NewInt(10000000000000), big.NewInt(10000000000000))}, }) chainID, err := contractBackend.ChainID(context.Background()) if err != nil { From f8a0de952087fbb3eae7189b67902a9abd6f490c Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 12 Jul 2023 00:03:30 +0700 Subject: [PATCH 70/82] Fix simulated EstimateGas --- accounts/abi/bind/backends/simulated.go | 33 ++++++------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index de87a6dda1..ca48f0eb95 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -431,32 +431,23 @@ func (b *SimulatedBackend) EstimateGas(ctx context.Context, call tomochain.CallM cap = hi // Create a helper to check if a gas allowance results in an executable transaction - executable := func(gas uint64) (bool, error) { + executable := func(gas uint64) bool { call.Gas = gas snapshot := b.pendingState.Snapshot() _, _, failed, err := b.callContract(ctx, call, b.pendingBlock, b.pendingState) + fmt.Println("EstimateGas", err, failed) b.pendingState.RevertToSnapshot(snapshot) - if err != nil { - if errors.Is(err, core.ErrIntrinsicGas) || errors.Is(err, vm.ErrCodeStoreOutOfGas) || errors.Is(err, vm.ErrOutOfGas) { - return true, nil // Special cases, raise gas limit - } - return true, err // Bail out + if err != nil || failed { + return false } - return false, err + return true } // Execute the binary search and hone in on an executable gas limit for lo+1 < hi { mid := (hi + lo) / 2 - failed, err := executable(mid) - // If the error is not nil(consensus error), it means the provided message - // call or transaction will never be accepted no matter how much gas it is - // assigned. Return the error directly, don't struggle anymore - if err != nil { - return 0, err - } - if failed { + if !executable(mid) { lo = mid } else { hi = mid @@ -464,16 +455,8 @@ func (b *SimulatedBackend) EstimateGas(ctx context.Context, call tomochain.CallM } // Reject the transaction as invalid if it still fails at the highest allowance if hi == cap { - failed, err := executable(hi) - if err != nil { - return 0, err - } - if failed { - if err != vm.ErrOutOfGas { - return 0, err - } - // Otherwise, the specified gas cap is too low - return 0, fmt.Errorf("gas required exceeds allowance (%d)", cap) + if !executable(hi) { + return 0, errGasEstimationFailed } } return hi, nil From 57eb63afdcb13e0b3f1efeaf5ef54d5f85dceefc Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 12 Jul 2023 00:23:44 +0700 Subject: [PATCH 71/82] Fix preState tracer unit tests --- eth/tracers/tracers_test.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index 9f469aeb89..c61ceaaa84 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -122,14 +122,21 @@ type callTracerTest struct { func TestPrestateTracerCreate2(t *testing.T) { common.TIPTomoXCancellationFee = big.NewInt(10000000000000) - unsignedTx := types.NewTransaction(1, common.HexToAddress("0x00000000000000000000000000000000deadbeef"), - new(big.Int), 5000000, big.NewInt(1), []byte{}) + to := common.HexToAddress("0x00000000000000000000000000000000deadbeef") + unsignedTx := types.NewTx(&types.LegacyTx{ + Nonce: 1, + To: &to, + Value: new(big.Int), + Gas: 5000000, + GasPrice: big.NewInt(1), + Data: []byte{}, + }) privateKeyECDSA, err := ecdsa.GenerateKey(crypto.S256(), rand.Reader) if err != nil { t.Fatalf("err %v", err) } - signer := types.NewEIP155Signer(big.NewInt(1)) + signer := types.LatestSignerForChainID(big.NewInt(1)) tx, err := types.SignTx(unsignedTx, signer, privateKeyECDSA) if err != nil { t.Fatalf("err %v", err) @@ -154,6 +161,7 @@ func TestPrestateTracerCreate2(t *testing.T) { Difficulty: big.NewInt(0x30000), GasLimit: uint64(6000000), GasPrice: big.NewInt(1), + BaseFee: big.NewInt(0), } alloc := core.GenesisAlloc{} @@ -177,7 +185,7 @@ func TestPrestateTracerCreate2(t *testing.T) { if err != nil { t.Fatalf("failed to create call tracer: %v", err) } - evm := vm.NewEVM(context, statedb, nil, params.MainnetChainConfig, vm.Config{Debug: true, Tracer: tracer}) + evm := vm.NewEVM(context, statedb, nil, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer, NoBaseFee: true}) msg, err := core.TransactionToMessage(tx, signer, nil, nil) if err != nil { From 8e7cf690b453b3f73bdb56b77fa724100c8eba68 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 12 Jul 2023 19:52:22 +0700 Subject: [PATCH 72/82] Fix call tracer tests --- core/state_transition.go | 2 +- eth/tracers/tracers_test.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/core/state_transition.go b/core/state_transition.go index f6e687b74c..211b16a51f 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -364,7 +364,7 @@ func (st *StateTransition) TransitionDb(owner common.Address) (ret []byte, usedG st.state.AddBalance(st.evm.Coinbase, new(big.Int).Mul(new(big.Int).SetUint64(st.gasUsed()), st.gasPrice)) } - return ret, st.gasUsed(), vmerr != nil, vmerr + return ret, st.gasUsed(), vmerr != nil, err } func (st *StateTransition) refundGas() { diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index c61ceaaa84..6acb78c27c 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -251,6 +251,7 @@ func TestCallTracer(t *testing.T) { Difficulty: (*big.Int)(test.Context.Difficulty), GasLimit: uint64(test.Context.GasLimit), GasPrice: tx.GasPrice(), + BaseFee: new(big.Int).SetUint64(0), } db := rawdb.NewMemoryDatabase() statedb := tests.MakePreState(db, test.Genesis.Alloc) From 6ce00c85d2feec03b53efd3a635f2d26269e3d3e Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 12 Jul 2023 21:11:57 +0700 Subject: [PATCH 73/82] Implement LRU common package --- common/lru/basiclru.go | 223 +++++++++++++++++++++++++++++++ common/lru/basiclru_test.go | 255 ++++++++++++++++++++++++++++++++++++ common/lru/blob_lru.go | 84 ++++++++++++ common/lru/blob_lru_test.go | 155 ++++++++++++++++++++++ common/lru/lru.go | 95 ++++++++++++++ 5 files changed, 812 insertions(+) create mode 100644 common/lru/basiclru.go create mode 100644 common/lru/basiclru_test.go create mode 100644 common/lru/blob_lru.go create mode 100644 common/lru/blob_lru_test.go create mode 100644 common/lru/lru.go diff --git a/common/lru/basiclru.go b/common/lru/basiclru.go new file mode 100644 index 0000000000..a429157fe5 --- /dev/null +++ b/common/lru/basiclru.go @@ -0,0 +1,223 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package lru implements generically-typed LRU caches. +package lru + +// BasicLRU is a simple LRU cache. +// +// This type is not safe for concurrent use. +// The zero value is not valid, instances must be created using NewCache. +type BasicLRU[K comparable, V any] struct { + list *list[K] + items map[K]cacheItem[K, V] + cap int +} + +type cacheItem[K any, V any] struct { + elem *listElem[K] + value V +} + +// NewBasicLRU creates a new LRU cache. +func NewBasicLRU[K comparable, V any](capacity int) BasicLRU[K, V] { + if capacity <= 0 { + capacity = 1 + } + c := BasicLRU[K, V]{ + items: make(map[K]cacheItem[K, V]), + list: newList[K](), + cap: capacity, + } + return c +} + +// Add adds a value to the cache. Returns true if an item was evicted to store the new item. +func (c *BasicLRU[K, V]) Add(key K, value V) (evicted bool) { + item, ok := c.items[key] + if ok { + // Already exists in cache. + item.value = value + c.items[key] = item + c.list.moveToFront(item.elem) + return false + } + + var elem *listElem[K] + if c.Len() >= c.cap { + elem = c.list.removeLast() + delete(c.items, elem.v) + evicted = true + } else { + elem = new(listElem[K]) + } + + // Store the new item. + // Note that, if another item was evicted, we re-use its list element here. + elem.v = key + c.items[key] = cacheItem[K, V]{elem, value} + c.list.pushElem(elem) + return evicted +} + +// Contains reports whether the given key exists in the cache. +func (c *BasicLRU[K, V]) Contains(key K) bool { + _, ok := c.items[key] + return ok +} + +// Get retrieves a value from the cache. This marks the key as recently used. +func (c *BasicLRU[K, V]) Get(key K) (value V, ok bool) { + item, ok := c.items[key] + if !ok { + return value, false + } + c.list.moveToFront(item.elem) + return item.value, true +} + +// GetOldest retrieves the least-recently-used item. +// Note that this does not update the item's recency. +func (c *BasicLRU[K, V]) GetOldest() (key K, value V, ok bool) { + lastElem := c.list.last() + if lastElem == nil { + return key, value, false + } + key = lastElem.v + item := c.items[key] + return key, item.value, true +} + +// Len returns the current number of items in the cache. +func (c *BasicLRU[K, V]) Len() int { + return len(c.items) +} + +// Peek retrieves a value from the cache, but does not mark the key as recently used. +func (c *BasicLRU[K, V]) Peek(key K) (value V, ok bool) { + item, ok := c.items[key] + return item.value, ok +} + +// Purge empties the cache. +func (c *BasicLRU[K, V]) Purge() { + c.list.init() + for k := range c.items { + delete(c.items, k) + } +} + +// Remove drops an item from the cache. Returns true if the key was present in cache. +func (c *BasicLRU[K, V]) Remove(key K) bool { + item, ok := c.items[key] + if ok { + delete(c.items, key) + c.list.remove(item.elem) + } + return ok +} + +// RemoveOldest drops the least recently used item. +func (c *BasicLRU[K, V]) RemoveOldest() (key K, value V, ok bool) { + lastElem := c.list.last() + if lastElem == nil { + return key, value, false + } + + key = lastElem.v + item := c.items[key] + delete(c.items, key) + c.list.remove(lastElem) + return key, item.value, true +} + +// Keys returns all keys in the cache. +func (c *BasicLRU[K, V]) Keys() []K { + keys := make([]K, 0, len(c.items)) + return c.list.appendTo(keys) +} + +// list is a doubly-linked list holding items of type he. +// The zero value is not valid, use newList to create lists. +type list[T any] struct { + root listElem[T] +} + +type listElem[T any] struct { + next *listElem[T] + prev *listElem[T] + v T +} + +func newList[T any]() *list[T] { + l := new(list[T]) + l.init() + return l +} + +// init reinitializes the list, making it empty. +func (l *list[T]) init() { + l.root.next = &l.root + l.root.prev = &l.root +} + +// push adds an element to the front of the list. +func (l *list[T]) pushElem(e *listElem[T]) { + e.prev = &l.root + e.next = l.root.next + l.root.next = e + e.next.prev = e +} + +// moveToFront makes 'node' the head of the list. +func (l *list[T]) moveToFront(e *listElem[T]) { + e.prev.next = e.next + e.next.prev = e.prev + l.pushElem(e) +} + +// remove removes an element from the list. +func (l *list[T]) remove(e *listElem[T]) { + e.prev.next = e.next + e.next.prev = e.prev + e.next, e.prev = nil, nil +} + +// removeLast removes the last element of the list. +func (l *list[T]) removeLast() *listElem[T] { + last := l.last() + if last != nil { + l.remove(last) + } + return last +} + +// last returns the last element of the list, or nil if the list is empty. +func (l *list[T]) last() *listElem[T] { + e := l.root.prev + if e == &l.root { + return nil + } + return e +} + +// appendTo appends all list elements to a slice. +func (l *list[T]) appendTo(slice []T) []T { + for e := l.root.prev; e != &l.root; e = e.prev { + slice = append(slice, e.v) + } + return slice +} diff --git a/common/lru/basiclru_test.go b/common/lru/basiclru_test.go new file mode 100644 index 0000000000..29812bda15 --- /dev/null +++ b/common/lru/basiclru_test.go @@ -0,0 +1,255 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package lru + +import ( + crand "crypto/rand" + "fmt" + "io" + "math/rand" + "testing" +) + +// Some of these test cases were adapted +// from https://github.com/hashicorp/golang-lru/blob/master/simplelru/lru_test.go + +func TestBasicLRU(t *testing.T) { + cache := NewBasicLRU[int, int](128) + + for i := 0; i < 256; i++ { + cache.Add(i, i) + } + if cache.Len() != 128 { + t.Fatalf("bad len: %v", cache.Len()) + } + + // Check that Keys returns least-recent key first. + keys := cache.Keys() + if len(keys) != 128 { + t.Fatal("wrong Keys() length", len(keys)) + } + for i, k := range keys { + v, ok := cache.Peek(k) + if !ok { + t.Fatalf("expected key %d be present", i) + } + if v != k { + t.Fatalf("expected %d == %d", k, v) + } + if v != i+128 { + t.Fatalf("wrong value at key %d: %d, want %d", i, v, i+128) + } + } + + for i := 0; i < 128; i++ { + _, ok := cache.Get(i) + if ok { + t.Fatalf("%d should be evicted", i) + } + } + for i := 128; i < 256; i++ { + _, ok := cache.Get(i) + if !ok { + t.Fatalf("%d should not be evicted", i) + } + } + + for i := 128; i < 192; i++ { + ok := cache.Remove(i) + if !ok { + t.Fatalf("%d should be in cache", i) + } + ok = cache.Remove(i) + if ok { + t.Fatalf("%d should not be in cache", i) + } + _, ok = cache.Get(i) + if ok { + t.Fatalf("%d should be deleted", i) + } + } + + // Request item 192. + cache.Get(192) + // It should be the last item returned by Keys(). + for i, k := range cache.Keys() { + if (i < 63 && k != i+193) || (i == 63 && k != 192) { + t.Fatalf("out of order key: %v", k) + } + } + + cache.Purge() + if cache.Len() != 0 { + t.Fatalf("bad len: %v", cache.Len()) + } + if _, ok := cache.Get(200); ok { + t.Fatalf("should contain nothing") + } +} + +func TestBasicLRUAddExistingKey(t *testing.T) { + cache := NewBasicLRU[int, int](1) + + cache.Add(1, 1) + cache.Add(1, 2) + + v, _ := cache.Get(1) + if v != 2 { + t.Fatal("wrong value:", v) + } +} + +// This test checks GetOldest and RemoveOldest. +func TestBasicLRUGetOldest(t *testing.T) { + cache := NewBasicLRU[int, int](128) + for i := 0; i < 256; i++ { + cache.Add(i, i) + } + + k, _, ok := cache.GetOldest() + if !ok { + t.Fatalf("missing") + } + if k != 128 { + t.Fatalf("bad: %v", k) + } + + k, _, ok = cache.RemoveOldest() + if !ok { + t.Fatalf("missing") + } + if k != 128 { + t.Fatalf("bad: %v", k) + } + + k, _, ok = cache.RemoveOldest() + if !ok { + t.Fatalf("missing oldest item") + } + if k != 129 { + t.Fatalf("wrong oldest item: %v", k) + } +} + +// Test that Add returns true/false if an eviction occurred +func TestBasicLRUAddReturnValue(t *testing.T) { + cache := NewBasicLRU[int, int](1) + if cache.Add(1, 1) { + t.Errorf("first add shouldn't have evicted") + } + if !cache.Add(2, 2) { + t.Errorf("second add should have evicted") + } +} + +// This test verifies that Contains doesn't change item recency. +func TestBasicLRUContains(t *testing.T) { + cache := NewBasicLRU[int, int](2) + cache.Add(1, 1) + cache.Add(2, 2) + if !cache.Contains(1) { + t.Errorf("1 should be in the cache") + } + cache.Add(3, 3) + if cache.Contains(1) { + t.Errorf("Contains should not have updated recency of 1") + } +} + +// Test that Peek doesn't update recent-ness +func TestBasicLRUPeek(t *testing.T) { + cache := NewBasicLRU[int, int](2) + cache.Add(1, 1) + cache.Add(2, 2) + if v, ok := cache.Peek(1); !ok || v != 1 { + t.Errorf("1 should be set to 1") + } + cache.Add(3, 3) + if cache.Contains(1) { + t.Errorf("should not have updated recent-ness of 1") + } +} + +func BenchmarkLRU(b *testing.B) { + var ( + capacity = 1000 + indexes = make([]int, capacity*20) + keys = make([]string, capacity) + values = make([][]byte, capacity) + ) + for i := range indexes { + indexes[i] = rand.Intn(capacity) + } + for i := range keys { + b := make([]byte, 32) + crand.Read(b) + keys[i] = string(b) + crand.Read(b) + values[i] = b + } + + var sink []byte + + b.Run("Add/BasicLRU", func(b *testing.B) { + cache := NewBasicLRU[int, int](capacity) + for i := 0; i < b.N; i++ { + cache.Add(i, i) + } + }) + b.Run("Get/BasicLRU", func(b *testing.B) { + cache := NewBasicLRU[string, []byte](capacity) + for i := 0; i < capacity; i++ { + index := indexes[i] + cache.Add(keys[index], values[index]) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + k := keys[indexes[i%len(indexes)]] + v, ok := cache.Get(k) + if ok { + sink = v + } + } + }) + + // // vs. github.com/hashicorp/golang-lru/simplelru + // b.Run("Add/simplelru.LRU", func(b *testing.B) { + // cache, _ := simplelru.NewLRU(capacity, nil) + // for i := 0; i < b.N; i++ { + // cache.Add(i, i) + // } + // }) + // b.Run("Get/simplelru.LRU", func(b *testing.B) { + // cache, _ := simplelru.NewLRU(capacity, nil) + // for i := 0; i < capacity; i++ { + // index := indexes[i] + // cache.Add(keys[index], values[index]) + // } + // + // b.ResetTimer() + // for i := 0; i < b.N; i++ { + // k := keys[indexes[i%len(indexes)]] + // v, ok := cache.Get(k) + // if ok { + // sink = v.([]byte) + // } + // } + // }) + + fmt.Fprintln(io.Discard, sink) +} diff --git a/common/lru/blob_lru.go b/common/lru/blob_lru.go new file mode 100644 index 0000000000..c9b3398503 --- /dev/null +++ b/common/lru/blob_lru.go @@ -0,0 +1,84 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package lru + +import ( + "math" + "sync" +) + +// blobType is the type constraint for values stored in SizeConstrainedCache. +type blobType interface { + ~[]byte | ~string +} + +// SizeConstrainedCache is a cache where capacity is in bytes (instead of item count). When the cache +// is at capacity, and a new item is added, older items are evicted until the size +// constraint is met. +// +// OBS: This cache assumes that items are content-addressed: keys are unique per content. +// In other words: two Add(..) with the same key K, will always have the same value V. +type SizeConstrainedCache[K comparable, V blobType] struct { + size uint64 + maxSize uint64 + lru BasicLRU[K, V] + lock sync.Mutex +} + +// NewSizeConstrainedCache creates a new size-constrained LRU cache. +func NewSizeConstrainedCache[K comparable, V blobType](maxSize uint64) *SizeConstrainedCache[K, V] { + return &SizeConstrainedCache[K, V]{ + size: 0, + maxSize: maxSize, + lru: NewBasicLRU[K, V](math.MaxInt), + } +} + +// Add adds a value to the cache. Returns true if an eviction occurred. +// OBS: This cache assumes that items are content-addressed: keys are unique per content. +// In other words: two Add(..) with the same key K, will always have the same value V. +// OBS: The value is _not_ copied on Add, so the caller must not modify it afterwards. +func (c *SizeConstrainedCache[K, V]) Add(key K, value V) (evicted bool) { + c.lock.Lock() + defer c.lock.Unlock() + + // Unless it is already present, might need to evict something. + // OBS: If it is present, we still call Add internally to bump the recentness. + if !c.lru.Contains(key) { + targetSize := c.size + uint64(len(value)) + for targetSize > c.maxSize { + evicted = true + _, v, ok := c.lru.RemoveOldest() + if !ok { + // list is now empty. Break + break + } + targetSize -= uint64(len(v)) + } + c.size = targetSize + } + c.lru.Add(key, value) + return evicted +} + +// Get looks up a key's value from the cache. +func (c *SizeConstrainedCache[K, V]) Get(key K) (V, bool) { + c.lock.Lock() + defer c.lock.Unlock() + + return c.lru.Get(key) +} diff --git a/common/lru/blob_lru_test.go b/common/lru/blob_lru_test.go new file mode 100644 index 0000000000..ca1b0ddd74 --- /dev/null +++ b/common/lru/blob_lru_test.go @@ -0,0 +1,155 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package lru + +import ( + "encoding/binary" + "fmt" + "testing" +) + +type testKey [8]byte + +func mkKey(i int) (key testKey) { + binary.LittleEndian.PutUint64(key[:], uint64(i)) + return key +} + +func TestSizeConstrainedCache(t *testing.T) { + lru := NewSizeConstrainedCache[testKey, []byte](100) + var want uint64 + // Add 11 items of 10 byte each. First item should be swapped out + for i := 0; i < 11; i++ { + k := mkKey(i) + v := fmt.Sprintf("value-%04d", i) + lru.Add(k, []byte(v)) + want += uint64(len(v)) + if want > 100 { + want = 100 + } + if have := lru.size; have != want { + t.Fatalf("size wrong, have %d want %d", have, want) + } + } + // Zero:th should be evicted + { + k := mkKey(0) + if _, ok := lru.Get(k); ok { + t.Fatalf("should be evicted: %v", k) + } + } + // Elems 1-11 should be present + for i := 1; i < 11; i++ { + k := mkKey(i) + want := fmt.Sprintf("value-%04d", i) + have, ok := lru.Get(k) + if !ok { + t.Fatalf("missing key %v", k) + } + if string(have) != want { + t.Fatalf("wrong value, have %v want %v", have, want) + } + } +} + +// This test adds inserting an element exceeding the max size. +func TestSizeConstrainedCacheOverflow(t *testing.T) { + lru := NewSizeConstrainedCache[testKey, []byte](100) + + // Add 10 items of 10 byte each, filling the cache + for i := 0; i < 10; i++ { + k := mkKey(i) + v := fmt.Sprintf("value-%04d", i) + lru.Add(k, []byte(v)) + } + // Add one single large elem. We expect it to swap out all entries. + { + k := mkKey(1337) + v := make([]byte, 200) + lru.Add(k, v) + } + // Elems 0-9 should be missing + for i := 1; i < 10; i++ { + k := mkKey(i) + if _, ok := lru.Get(k); ok { + t.Fatalf("should be evicted: %v", k) + } + } + // The size should be accurate + if have, want := lru.size, uint64(200); have != want { + t.Fatalf("size wrong, have %d want %d", have, want) + } + // Adding one small item should swap out the large one + { + i := 0 + k := mkKey(i) + v := fmt.Sprintf("value-%04d", i) + lru.Add(k, []byte(v)) + if have, want := lru.size, uint64(10); have != want { + t.Fatalf("size wrong, have %d want %d", have, want) + } + } +} + +// This checks what happens when inserting the same k/v multiple times. +func TestSizeConstrainedCacheSameItem(t *testing.T) { + lru := NewSizeConstrainedCache[testKey, []byte](100) + + // Add one 10 byte-item 10 times. + k := mkKey(0) + v := fmt.Sprintf("value-%04d", 0) + for i := 0; i < 10; i++ { + lru.Add(k, []byte(v)) + } + + // The size should be accurate. + if have, want := lru.size, uint64(10); have != want { + t.Fatalf("size wrong, have %d want %d", have, want) + } +} + +// This tests that empty/nil values are handled correctly. +func TestSizeConstrainedCacheEmpties(t *testing.T) { + lru := NewSizeConstrainedCache[testKey, []byte](100) + + // This test abuses the lru a bit, using different keys for identical value(s). + for i := 0; i < 10; i++ { + lru.Add(testKey{byte(i)}, []byte{}) + lru.Add(testKey{byte(255 - i)}, nil) + } + + // The size should not count, only the values count. So this could be a DoS + // since it basically has no cap, and it is intentionally overloaded with + // different-keyed 0-length values. + if have, want := lru.size, uint64(0); have != want { + t.Fatalf("size wrong, have %d want %d", have, want) + } + + for i := 0; i < 10; i++ { + if v, ok := lru.Get(testKey{byte(i)}); !ok { + t.Fatalf("test %d: expected presence", i) + } else if v == nil { + t.Fatalf("test %d, v is nil", i) + } + + if v, ok := lru.Get(testKey{byte(255 - i)}); !ok { + t.Fatalf("test %d: expected presence", i) + } else if v != nil { + t.Fatalf("test %d, v is not nil", i) + } + } +} diff --git a/common/lru/lru.go b/common/lru/lru.go new file mode 100644 index 0000000000..45965adb0d --- /dev/null +++ b/common/lru/lru.go @@ -0,0 +1,95 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package lru + +import "sync" + +// Cache is a LRU cache. +// This type is safe for concurrent use. +type Cache[K comparable, V any] struct { + cache BasicLRU[K, V] + mu sync.Mutex +} + +// NewCache creates an LRU cache. +func NewCache[K comparable, V any](capacity int) *Cache[K, V] { + return &Cache[K, V]{cache: NewBasicLRU[K, V](capacity)} +} + +// Add adds a value to the cache. Returns true if an item was evicted to store the new item. +func (c *Cache[K, V]) Add(key K, value V) (evicted bool) { + c.mu.Lock() + defer c.mu.Unlock() + + return c.cache.Add(key, value) +} + +// Contains reports whether the given key exists in the cache. +func (c *Cache[K, V]) Contains(key K) bool { + c.mu.Lock() + defer c.mu.Unlock() + + return c.cache.Contains(key) +} + +// Get retrieves a value from the cache. This marks the key as recently used. +func (c *Cache[K, V]) Get(key K) (value V, ok bool) { + c.mu.Lock() + defer c.mu.Unlock() + + return c.cache.Get(key) +} + +// Len returns the current number of items in the cache. +func (c *Cache[K, V]) Len() int { + c.mu.Lock() + defer c.mu.Unlock() + + return c.cache.Len() +} + +// Peek retrieves a value from the cache, but does not mark the key as recently used. +func (c *Cache[K, V]) Peek(key K) (value V, ok bool) { + c.mu.Lock() + defer c.mu.Unlock() + + return c.cache.Peek(key) +} + +// Purge empties the cache. +func (c *Cache[K, V]) Purge() { + c.mu.Lock() + defer c.mu.Unlock() + + c.cache.Purge() +} + +// Remove drops an item from the cache. Returns true if the key was present in cache. +func (c *Cache[K, V]) Remove(key K) bool { + c.mu.Lock() + defer c.mu.Unlock() + + return c.cache.Remove(key) +} + +// Keys returns all keys of items currently in the LRU. +func (c *Cache[K, V]) Keys() []K { + c.mu.Lock() + defer c.mu.Unlock() + + return c.cache.Keys() +} From df825bbb2744ed077aaca9e80b903b0a10e84a23 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 12 Jul 2023 21:15:43 +0700 Subject: [PATCH 74/82] Add CacheLogSizeFlag --- cmd/tomo/main.go | 1 + cmd/utils/flags.go | 14 +++++++++++--- eth/config.go | 19 +++++++++++-------- eth/gen_config.go | 6 ++++++ 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/cmd/tomo/main.go b/cmd/tomo/main.go index b39eafa95e..47ea5c5e03 100644 --- a/cmd/tomo/main.go +++ b/cmd/tomo/main.go @@ -93,6 +93,7 @@ var ( //utils.CacheDatabaseFlag, //utils.CacheGCFlag, //utils.TrieCacheGenFlag, + utils.CacheLogSizeFlag, utils.ListenPortFlag, utils.MaxPeersFlag, utils.MaxPendingPeersFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 09e46bb28f..2b0f32274f 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -305,6 +305,11 @@ var ( Usage: "Percentage of cache memory allowance to use for trie pruning", Value: 25, } + CacheLogSizeFlag = &cli.IntFlag{ + Name: "cache.blocklogs", + Usage: "Size (in number of blocks) of the log cache for filtering", + Value: eth.DefaultConfig.FilterLogCacheSize, + } // Miner settings StakingEnabledFlag = cli.BoolFlag{ Name: "mine", @@ -1141,13 +1146,16 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) { cfg.TrieCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 } + if ctx.GlobalIsSet(CacheLogSizeFlag.Name) { + cfg.FilterLogCacheSize = ctx.Int(CacheLogSizeFlag.Name) + } if ctx.GlobalIsSet(StakerThreadsFlag.Name) { cfg.MinerThreads = ctx.GlobalInt(StakerThreadsFlag.Name) } if ctx.GlobalIsSet(DocRootFlag.Name) { cfg.DocRoot = ctx.GlobalString(DocRootFlag.Name) } - if ctx.IsSet(RPCGlobalGasCapFlag.Name) { + if ctx.GlobalIsSet(RPCGlobalGasCapFlag.Name) { cfg.RPCGasCap = ctx.Uint64(RPCGlobalGasCapFlag.Name) } if cfg.RPCGasCap != 0 { @@ -1155,10 +1163,10 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { } else { log.Info("Global gas cap disabled") } - if ctx.IsSet(RPCGlobalEVMTimeoutFlag.Name) { + if ctx.GlobalIsSet(RPCGlobalEVMTimeoutFlag.Name) { cfg.RPCEVMTimeout = ctx.Duration(RPCGlobalEVMTimeoutFlag.Name) } - if ctx.IsSet(RPCGlobalTxFeeCapFlag.Name) { + if ctx.GlobalIsSet(RPCGlobalTxFeeCapFlag.Name) { cfg.RPCTxFeeCap = ctx.Float64(RPCGlobalTxFeeCapFlag.Name) } if ctx.GlobalIsSet(ExtraDataFlag.Name) { diff --git a/eth/config.go b/eth/config.go index 7390feec47..704756bd4a 100644 --- a/eth/config.go +++ b/eth/config.go @@ -43,14 +43,14 @@ var DefaultConfig = Config{ DatasetsInMem: 1, DatasetsOnDisk: 2, }, - NetworkId: 88, - LightPeers: 100, - DatabaseCache: 768, - TrieCache: 256, - TrieTimeout: 5 * time.Minute, - GasPrice: big.NewInt(0.25 * params.Shannon), - - TxPool: core.DefaultTxPoolConfig, + NetworkId: 88, + LightPeers: 100, + DatabaseCache: 768, + TrieCache: 256, + TrieTimeout: 5 * time.Minute, + GasPrice: big.NewInt(0.25 * params.Shannon), + FilterLogCacheSize: 32, + TxPool: core.DefaultTxPoolConfig, GPO: gasprice.Config{ Blocks: 20, Percentile: 60, @@ -101,6 +101,9 @@ type Config struct { TrieCache int TrieTimeout time.Duration + // This is the number of blocks for which logs will be cached in the filter system. + FilterLogCacheSize int + // Mining-related options Etherbase common.Address `toml:",omitempty"` MinerThreads int `toml:",omitempty"` diff --git a/eth/gen_config.go b/eth/gen_config.go index 514b70170e..170ebdf3d2 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -30,6 +30,7 @@ func (c Config) MarshalTOML() (interface{}, error) { DatabaseCache int TrieCache int TrieTimeout time.Duration + FilterLogCacheSize int Etherbase common.Address `toml:",omitempty"` MinerThreads int `toml:",omitempty"` ExtraData hexutil.Bytes `toml:",omitempty"` @@ -55,6 +56,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.DatabaseCache = c.DatabaseCache enc.TrieCache = c.TrieCache enc.TrieTimeout = c.TrieTimeout + enc.FilterLogCacheSize = c.FilterLogCacheSize enc.Etherbase = c.Etherbase enc.MinerThreads = c.MinerThreads enc.ExtraData = c.ExtraData @@ -84,6 +86,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { DatabaseCache *int TrieCache *int TrieTimeout *time.Duration + FilterLogCacheSize *int Etherbase *common.Address `toml:",omitempty"` MinerThreads *int `toml:",omitempty"` ExtraData *hexutil.Bytes `toml:",omitempty"` @@ -134,6 +137,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.TrieTimeout != nil { c.TrieTimeout = *dec.TrieTimeout } + if dec.FilterLogCacheSize != nil { + c.FilterLogCacheSize = *dec.FilterLogCacheSize + } if dec.Etherbase != nil { c.Etherbase = *dec.Etherbase } From 242c5ebb38eb4788d6dcf0711a0b7196ad5b48bc Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 12 Jul 2023 21:22:32 +0700 Subject: [PATCH 75/82] Update filters package and unit tests --- accounts/abi/bind/backends/simulated.go | 84 +++++-- core/chain_makers.go | 22 ++ core/database_util.go | 81 +++++- eth/api_backend.go | 46 +++- eth/backend.go | 6 +- eth/filters/api.go | 109 ++++---- eth/filters/bench_test.go | 31 +-- eth/filters/filter.go | 316 ++++++++++++++++------- eth/filters/filter_system.go | 120 +++++++-- eth/filters/filter_system_test.go | 87 +++++-- eth/filters/filter_test.go | 319 +++++++++++++++--------- event/event.go | 7 + interfaces.go | 1 + les/api_backend.go | 17 +- les/backend.go | 5 +- miner/miner.go | 14 +- miner/worker.go | 8 + 17 files changed, 928 insertions(+), 345 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index ca48f0eb95..fe68fa8c7e 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -59,11 +59,13 @@ type SimulatedBackend struct { database ethdb.Database // In memory database to store our testing data blockchain *core.BlockChain // Ethereum blockchain to handle the consensus - mu sync.Mutex - pendingBlock *types.Block // Currently pending block that will be imported on request - pendingState *state.StateDB // Currently pending state that will be the active on request + mu sync.Mutex + pendingBlock *types.Block // Currently pending block that will be imported on request + pendingState *state.StateDB // Currently pending state that will be the active on request + pendingReceipts types.Receipts // Currently receipts for the pending block - events *filters.EventSystem // Event system for filtering log events live + events *filters.EventSystem // Event system for filtering log events live + filterSystem *filters.FilterSystem // for filtering database logs config *params.ChainConfig } @@ -95,7 +97,9 @@ func NewSimulatedBackendWithChainConfig(alloc core.GenesisAlloc, config *params. blockchain: blockchain, config: genesis.Config, } - backend.events = filters.NewEventSystem(new(event.TypeMux), &filterBackend{database, blockchain, backend}, false) + filterBackend := &filterBackend{database, blockchain, backend} + backend.filterSystem = filters.NewFilterSystem(filterBackend, filters.Config{}) + backend.events = filters.NewEventSystem(backend.filterSystem, false) header := backend.blockchain.CurrentBlock() block := backend.blockchain.GetBlock(header.Hash(), header.Number().Uint64()) @@ -573,25 +577,31 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa // // TODO(karalabe): Deprecate when the subscription one can return past data too. func (b *SimulatedBackend) FilterLogs(ctx context.Context, query tomochain.FilterQuery) ([]types.Log, error) { - // Initialize unset filter boundaried to run from genesis to chain head - from := int64(0) - if query.FromBlock != nil { - from = query.FromBlock.Int64() - } - to := int64(-1) - if query.ToBlock != nil { - to = query.ToBlock.Int64() + var filter *filters.Filter + if query.BlockHash != nil { + // Block filter requested, construct a single-shot filter + filter = b.filterSystem.NewBlockFilter(*query.BlockHash, query.Addresses, query.Topics) + } else { + // Initialize unset filter boundaries to run from genesis to chain head + from := int64(0) + if query.FromBlock != nil { + from = query.FromBlock.Int64() + } + to := int64(-1) + if query.ToBlock != nil { + to = query.ToBlock.Int64() + } + // Construct the range filter + filter = b.filterSystem.NewRangeFilter(from, to, query.Addresses, query.Topics) } - // Construct and execute the filter - filter := filters.New(&filterBackend{b.database, b.blockchain, b}, from, to, query.Addresses, query.Topics) - + // Run the filter and return all the logs logs, err := filter.Logs(ctx) if err != nil { return nil, err } res := make([]types.Log, len(logs)) - for i, log := range logs { - res[i] = *log + for i, nLog := range logs { + res[i] = *nLog } return res, nil } @@ -671,19 +681,27 @@ func (fb *filterBackend) HeaderByNumber(ctx context.Context, block rpc.BlockNumb return fb.bc.GetHeaderByNumber(uint64(block.Int64())), nil } +func (fb *filterBackend) HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) { + return fb.bc.GetHeaderByHash(hash), nil +} + +func (fb *filterBackend) GetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error) { + if body := fb.bc.GetBody(hash); body != nil { + return body, nil + } + return nil, errors.New("block body not found") +} + +func (fb *filterBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { + return fb.backend.pendingBlock, fb.backend.pendingReceipts +} + func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { return core.GetBlockReceipts(fb.db, hash, core.GetBlockNumber(fb.db, hash), fb.ChainConfig()), nil } -func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { - receipts := core.GetBlockReceipts(fb.db, hash, core.GetBlockNumber(fb.db, hash), fb.bc.Config()) - if receipts == nil { - return nil, nil - } - logs := make([][]*types.Log, len(receipts)) - for i, receipt := range receipts { - logs[i] = receipt.Logs - } +func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash, number uint64) ([][]*types.Log, error) { + logs := core.ReadLogs(fb.db, hash, number, fb.bc.Config()) return logs, nil } @@ -702,6 +720,9 @@ func (fb *filterBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEve func (fb *filterBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { return fb.bc.SubscribeLogsEvent(ch) } +func (fb *filterBackend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { + return nullSubscription() +} func (fb *filterBackend) BloomStatus() (uint64, uint64) { return 4096, 0 } func (fb *filterBackend) ServiceFilter(ctx context.Context, ms *bloombits.MatcherSession) { @@ -710,3 +731,12 @@ func (fb *filterBackend) ServiceFilter(ctx context.Context, ms *bloombits.Matche func (fb *filterBackend) ChainConfig() *params.ChainConfig { panic("not supported") } +func (fb *filterBackend) CurrentHeader() *types.Header { + panic("not supported") +} +func nullSubscription() event.Subscription { + return event.NewSubscription(func(quit <-chan struct{}) error { + <-quit + return nil + }) +} diff --git a/core/chain_makers.go b/core/chain_makers.go index 3499368f8c..2af994d254 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -86,6 +86,15 @@ func (b *BlockGen) AddTx(tx *types.Transaction) { b.AddTxWithChain(nil, tx) } +// AddUncheckedTx forcefully adds a transaction to the block without any +// validation. +// +// AddUncheckedTx will cause consensus failures when used during real +// chain processing. This is best used in conjunction with raw block insertion. +func (b *BlockGen) AddUncheckedTx(tx *types.Transaction) { + b.txs = append(b.txs, tx) +} + // AddTxWithChain adds a transaction to the generated block. If no coinbase has // been set, the block's coinbase is set to the zero address. // @@ -263,6 +272,19 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse return blocks, receipts } +// GenerateChainWithGenesis is a wrapper of GenerateChain which will initialize +// genesis block to database first according to the provided genesis specification +// then generate chain on top. +func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gen func(int, *BlockGen)) (ethdb.Database, []*types.Block, []types.Receipts) { + db := rawdb.NewMemoryDatabase() + _, err := genesis.Commit(db) + if err != nil { + panic(err) + } + blocks, receipts := GenerateChain(genesis.Config, genesis.ToBlock(db), engine, db, n, gen) + return db, blocks, receipts +} + func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.StateDB, engine consensus.Engine) *types.Header { var time *big.Int if parent.Time() == nil { diff --git a/core/database_util.go b/core/database_util.go index b1a0025ac8..4272ea6340 100644 --- a/core/database_util.go +++ b/core/database_util.go @@ -224,6 +224,75 @@ func GetTd(db DatabaseReader, hash common.Hash, number uint64) *big.Int { return td } +// storedReceiptRLP is the storage encoding of a receipt. +// Re-definition in core/types/receipt.go. +// TODO: Re-use the existing definition. +type storedReceiptRLP struct { + PostStateOrStatus []byte + CumulativeGasUsed uint64 + Logs []*types.Log +} + +// ReceiptLogs is a barebone version of ReceiptForStorage which only keeps +// the list of logs. When decoding a stored receipt into this object we +// avoid creating the bloom filter. +type receiptLogs struct { + Logs []*types.Log +} + +// DecodeRLP implements rlp.Decoder. +func (r *receiptLogs) DecodeRLP(s *rlp.Stream) error { + var stored storedReceiptRLP + if err := s.Decode(&stored); err != nil { + return err + } + r.Logs = stored.Logs + return nil +} + +// DeriveLogFields fills the logs in receiptLogs with information such as block number, txhash, etc. +func deriveLogFields(receipts []*receiptLogs, hash common.Hash, number uint64, txs types.Transactions) error { + logIndex := uint(0) + if len(txs) != len(receipts) { + return errors.New("transaction and receipt count mismatch") + } + for i := 0; i < len(receipts); i++ { + txHash := txs[i].Hash() + // The derived log fields can simply be set from the block and transaction + for j := 0; j < len(receipts[i].Logs); j++ { + receipts[i].Logs[j].BlockNumber = number + receipts[i].Logs[j].BlockHash = hash + receipts[i].Logs[j].TxHash = txHash + receipts[i].Logs[j].TxIndex = uint(i) + receipts[i].Logs[j].Index = logIndex + logIndex++ + } + } + return nil +} + +// ReadLogs retrieves the logs for all transactions in a block. In case +// receipts is not found, a nil is returned. +// Note: ReadLogs does not derive unstored log fields. +func ReadLogs(db ethdb.Reader, hash common.Hash, number uint64, config *params.ChainConfig) [][]*types.Log { + // Retrieve the flattened receipt slice + data := ReadReceiptsRLP(db, hash, number) + if len(data) == 0 { + return nil + } + var receipts []*receiptLogs + if err := rlp.DecodeBytes(data, &receipts); err != nil { + log.Error("Invalid receipt array RLP", "hash", hash, "err", err) + return nil + } + + logs := make([][]*types.Log, len(receipts)) + for i, receipt := range receipts { + logs[i] = receipt.Logs + } + return logs +} + // GetBlock retrieves an entire block corresponding to the hash, assembling it // back from the stored header and body. If either the header or body could not // be retrieved nil is returned. @@ -244,11 +313,21 @@ func GetBlock(db DatabaseReader, hash common.Hash, number uint64) *types.Block { return types.NewBlockWithHeader(header).WithBody(body.Transactions, body.Uncles) } +// ReadReceiptsRLP retrieves all the transaction receipts belonging to a block in RLP encoding. +func ReadReceiptsRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { + data, _ := db.Get(append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash[:]...)) + if len(data) == 0 { + return nil + } + return data +} + // ReadRawReceipts retrieves all the transaction receipts belonging to a block. // The receipt metadata fields are not guaranteed to be populated, so they // should not be used. Use ReadReceipts instead if the metadata is needed. func ReadRawReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Receipts { - data, _ := db.Get(append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash[:]...)) + // Retrieve the flattened receipt slice + data := ReadReceiptsRLP(db, hash, number) if len(data) == 0 { return nil } diff --git a/eth/api_backend.go b/eth/api_backend.go index ed3768dd27..815b1e22e4 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -120,6 +120,46 @@ func (b *EthApiBackend) BlockByNumber(ctx context.Context, blockNr rpc.BlockNumb return b.eth.blockchain.GetBlockByNumber(uint64(blockNr)), nil } +func (b *EthApiBackend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) { + return b.eth.blockchain.GetBlockByHash(hash), nil +} + +// GetBody returns body of a block. It does not resolve special block numbers. +func (b *EthApiBackend) GetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error) { + if number < 0 || hash == (common.Hash{}) { + return nil, errors.New("invalid arguments; expect hash and no special block numbers") + } + if body := b.eth.blockchain.GetBody(hash); body != nil { + return body, nil + } + return nil, errors.New("block body not found") +} + +func (b *EthApiBackend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error) { + if blockNr, ok := blockNrOrHash.Number(); ok { + return b.BlockByNumber(ctx, blockNr) + } + if hash, ok := blockNrOrHash.Hash(); ok { + header := b.eth.blockchain.GetHeaderByHash(hash) + if header == nil { + return nil, errors.New("header for hash not found") + } + if blockNrOrHash.RequireCanonical && core.GetCanonicalHash(b.ChainDb(), header.Number.Uint64()) != hash { + return nil, errors.New("hash is not currently canonical") + } + block := b.eth.blockchain.GetBlock(hash, header.Number.Uint64()) + if block == nil { + return nil, errors.New("header found, but block body is missing") + } + return block, nil + } + return nil, errors.New("invalid arguments; neither block nor hash specified") +} + +func (b *EthApiBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { + return b.eth.miner.PendingBlockAndReceipts() +} + func (b *EthApiBackend) StateAndHeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*state.StateDB, *types.Header, error) { // Pending state is only known by the miner if blockNr == rpc.PendingBlockNumber { @@ -164,7 +204,7 @@ func (b *EthApiBackend) GetReceipts(ctx context.Context, blockHash common.Hash) return core.GetBlockReceipts(b.eth.chainDb, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash), b.ChainConfig()), nil } -func (b *EthApiBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) { +func (b *EthApiBackend) GetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error) { receipts := core.GetBlockReceipts(b.eth.chainDb, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash), b.ChainConfig()) if receipts == nil { return nil, nil @@ -257,6 +297,10 @@ func (b *EthApiBackend) OrderStats() (pending int, queued int) { return b.eth.txPool.Stats() } +func (b *EthApiBackend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { + return b.eth.miner.SubscribePendingLogs(ch) +} + func (b *EthApiBackend) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription { return b.eth.TxPool().SubscribeTxPreEvent(ch) } diff --git a/eth/backend.go b/eth/backend.go index 412c67d230..bcc2c21443 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -662,6 +662,10 @@ func (s *Ethereum) APIs() []rpc.API { // Append any APIs exposed explicitly by the consensus engine apis = append(apis, s.engine.APIs(s.BlockChain())...) + filterSystem := filters.NewFilterSystem(s.ApiBackend, filters.Config{ + LogCacheSize: s.config.FilterLogCacheSize, + }) + // Append all the local APIs and return return append(apis, []rpc.API{ { @@ -687,7 +691,7 @@ func (s *Ethereum) APIs() []rpc.API { }, { Namespace: "eth", Version: "1.0", - Service: filters.NewPublicFilterAPI(s.ApiBackend, false), + Service: filters.NewPublicFilterAPI(filterSystem, false), Public: true, }, { Namespace: "admin", diff --git a/eth/filters/api.go b/eth/filters/api.go index 72c811313e..5f2b69a04e 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -25,7 +25,7 @@ import ( "sync" "time" - ethereum "github.com/tomochain/tomochain" + "github.com/tomochain/tomochain" "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/common/hexutil" "github.com/tomochain/tomochain/core/types" @@ -35,7 +35,9 @@ import ( ) var ( - deadline = 5 * time.Minute // consider a filter inactive if it has not been polled for within deadline + deadline = 5 * time.Minute // consider a filter inactive if it has not been polled for within deadline + errInvalidTopic = errors.New("invalid topic(s)") + errFilterNotFound = errors.New("filter not found") ) // filter is a helper struct that holds meta information over the filter type @@ -52,46 +54,56 @@ type filter struct { // PublicFilterAPI offers support to create and manage filters. This will allow external clients to retrieve various // information related to the Ethereum protocol such als blocks, transactions and logs. type PublicFilterAPI struct { - backend Backend + sys *FilterSystem mux *event.TypeMux quit chan struct{} chainDb ethdb.Database events *EventSystem filtersMu sync.Mutex filters map[rpc.ID]*filter + timeout time.Duration } // NewPublicFilterAPI returns a new PublicFilterAPI instance. -func NewPublicFilterAPI(backend Backend, lightMode bool) *PublicFilterAPI { +func NewPublicFilterAPI(system *FilterSystem, lightMode bool) *PublicFilterAPI { api := &PublicFilterAPI{ - backend: backend, - mux: backend.EventMux(), - chainDb: backend.ChainDb(), - events: NewEventSystem(backend.EventMux(), backend, lightMode), + sys: system, + events: NewEventSystem(system, lightMode), filters: make(map[rpc.ID]*filter), + timeout: system.cfg.Timeout, } - go api.timeoutLoop() + go api.timeoutLoop(system.cfg.Timeout) return api } -// timeoutLoop runs every 5 minutes and deletes filters that have not been recently used. -// Tt is started when the api is created. -func (api *PublicFilterAPI) timeoutLoop() { - ticker := time.NewTicker(5 * time.Minute) +// timeoutLoop runs at the interval set by 'timeout' and deletes filters +// that have not been recently used. It is started when the API is created. +func (api *PublicFilterAPI) timeoutLoop(timeout time.Duration) { + var toUninstall []*Subscription + ticker := time.NewTicker(timeout) + defer ticker.Stop() for { <-ticker.C api.filtersMu.Lock() for id, f := range api.filters { select { case <-f.deadline.C: - f.s.Unsubscribe() + toUninstall = append(toUninstall, f.s) delete(api.filters, id) default: continue } } api.filtersMu.Unlock() + + // Unsubscribes are processed outside the lock to avoid the following scenario: + // event loop attempts broadcasting events to still active filters while + // Unsubscribe is waiting for it to process the uninstallation request. + for _, s := range toUninstall { + s.Unsubscribe() + } + toUninstall = nil } } @@ -241,7 +253,7 @@ func (api *PublicFilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc matchedLogs = make(chan []*types.Log) ) - logsSub, err := api.events.SubscribeLogs(ethereum.FilterQuery(crit), matchedLogs) + logsSub, err := api.events.SubscribeLogs(tomochain.FilterQuery(crit), matchedLogs) if err != nil { return nil, err } @@ -268,14 +280,8 @@ func (api *PublicFilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc } // FilterCriteria represents a request to create a new filter. -// -// TODO(karalabe): Kill this in favor of ethereum.FilterQuery. -type FilterCriteria struct { - FromBlock *big.Int - ToBlock *big.Int - Addresses []common.Address - Topics [][]common.Hash -} +// Same as tomochain.FilterQuery but with UnmarshalJSON() method. +type FilterCriteria tomochain.FilterQuery // NewFilter creates a new filter and returns the filter id. It can be // used to retrieve logs when the state changes. This method cannot be @@ -292,7 +298,7 @@ type FilterCriteria struct { // https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_newfilter func (api *PublicFilterAPI) NewFilter(crit FilterCriteria) (rpc.ID, error) { logs := make(chan []*types.Log) - logsSub, err := api.events.SubscribeLogs(ethereum.FilterQuery(crit), logs) + logsSub, err := api.events.SubscribeLogs(tomochain.FilterQuery(crit), logs) if err != nil { return rpc.ID(""), err } @@ -326,16 +332,24 @@ func (api *PublicFilterAPI) NewFilter(crit FilterCriteria) (rpc.ID, error) { // // https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getlogs func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*types.Log, error) { - // Convert the RPC block numbers into internal representations - if crit.FromBlock == nil { - crit.FromBlock = big.NewInt(rpc.LatestBlockNumber.Int64()) - } - if crit.ToBlock == nil { - crit.ToBlock = big.NewInt(rpc.LatestBlockNumber.Int64()) + var filter *Filter + if crit.BlockHash != nil { + // Block filter requested, construct a single-shot filter + filter = api.sys.NewBlockFilter(*crit.BlockHash, crit.Addresses, crit.Topics) + } else { + // Convert the RPC block numbers into internal representations + begin := rpc.LatestBlockNumber.Int64() + if crit.FromBlock != nil { + begin = crit.FromBlock.Int64() + } + end := rpc.LatestBlockNumber.Int64() + if crit.ToBlock != nil { + end = crit.ToBlock.Int64() + } + // Construct the range filter + filter = api.sys.NewRangeFilter(begin, end, crit.Addresses, crit.Topics) } - // Create and run the filter to get all the logs - filter := New(api.backend, crit.FromBlock.Int64(), crit.ToBlock.Int64(), crit.Addresses, crit.Topics) - + // Run the filter and return all the logs logs, err := filter.Logs(ctx) if err != nil { return nil, err @@ -370,20 +384,27 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty api.filtersMu.Unlock() if !found || f.typ != LogsSubscription { - return nil, fmt.Errorf("filter not found") + return nil, errFilterNotFound } - begin := rpc.LatestBlockNumber.Int64() - if f.crit.FromBlock != nil { - begin = f.crit.FromBlock.Int64() - } - end := rpc.LatestBlockNumber.Int64() - if f.crit.ToBlock != nil { - end = f.crit.ToBlock.Int64() + var filter *Filter + if f.crit.BlockHash != nil { + // Block filter requested, construct a single-shot filter + filter = api.sys.NewBlockFilter(*f.crit.BlockHash, f.crit.Addresses, f.crit.Topics) + } else { + // Convert the RPC block numbers into internal representations + begin := rpc.LatestBlockNumber.Int64() + if f.crit.FromBlock != nil { + begin = f.crit.FromBlock.Int64() + } + end := rpc.LatestBlockNumber.Int64() + if f.crit.ToBlock != nil { + end = f.crit.ToBlock.Int64() + } + // Construct the range filter + filter = api.sys.NewRangeFilter(begin, end, f.crit.Addresses, f.crit.Topics) } - // Create and run the filter to get all the logs - filter := New(api.backend, begin, end, f.crit.Addresses, f.crit.Topics) - + // Run the filter and return all the logs logs, err := filter.Logs(ctx) if err != nil { return nil, err diff --git a/eth/filters/bench_test.go b/eth/filters/bench_test.go index 3648a3db2f..cbf58e1c12 100644 --- a/eth/filters/bench_test.go +++ b/eth/filters/bench_test.go @@ -20,7 +20,6 @@ import ( "bytes" "context" "fmt" - "github.com/tomochain/tomochain/core/rawdb" "testing" "time" @@ -28,9 +27,9 @@ import ( "github.com/tomochain/tomochain/common/bitutil" "github.com/tomochain/tomochain/core" "github.com/tomochain/tomochain/core/bloombits" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/ethdb" - "github.com/tomochain/tomochain/event" "github.com/tomochain/tomochain/node" ) @@ -68,7 +67,7 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { benchDataDir := node.DefaultDataDir() + "/geth/chaindata" fmt.Println("Running bloombits benchmark section size:", sectionSize) - db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024,"") + db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "") if err != nil { b.Fatalf("error opening database at %v: %v", benchDataDir, err) } @@ -124,21 +123,24 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { fmt.Println("Running filter benchmarks...") start = time.Now() - mux := new(event.TypeMux) - var backend *testBackend + var ( + backend *testBackend + sys *FilterSystem + ) for i := 0; i < benchFilterCnt; i++ { if i%20 == 0 { db.Close() - db, _ = rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024,"") - backend = &testBackend{mux, db, cnt, new(event.Feed), new(event.Feed), new(event.Feed), new(event.Feed)} + db, _ = rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "") + backend = &testBackend{db: db, sections: cnt} + sys = NewFilterSystem(backend, Config{}) } var addr common.Address addr[0] = byte(i) addr[1] = byte(i / 256) - filter := New(backend, 0, int64(cnt*sectionSize-1), []common.Address{addr}, nil) + filter := sys.NewRangeFilter(0, int64(cnt*sectionSize-1), []common.Address{addr}, nil) if _, err := filter.Logs(context.Background()); err != nil { - b.Error("filter.Find error:", err) + b.Error("filter.Logs error:", err) } } d = time.Since(start) @@ -148,7 +150,7 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { } func forEachKey(db ethdb.Database, startPrefix, endPrefix []byte, fn func(key []byte)) { - it := db.NewIterator(startPrefix,nil) + it := db.NewIterator(startPrefix, nil) for it.Next() { key := it.Key() cmpLen := len(key) @@ -176,7 +178,7 @@ func clearBloomBits(db ethdb.Database) { func BenchmarkNoBloomBits(b *testing.B) { benchDataDir := node.DefaultDataDir() + "/geth/chaindata" fmt.Println("Running benchmark without bloombits") - db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024,"") + db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "") if err != nil { b.Fatalf("error opening database at %v: %v", benchDataDir, err) } @@ -188,11 +190,10 @@ func BenchmarkNoBloomBits(b *testing.B) { clearBloomBits(db) - fmt.Println("Running filter benchmarks...") + _, sys := newTestFilterSystem(b, db, Config{}) + b.Log("Running filter benchmarks...") start := time.Now() - mux := new(event.TypeMux) - backend := &testBackend{mux, db, 0, new(event.Feed), new(event.Feed), new(event.Feed), new(event.Feed)} - filter := New(backend, 0, int64(headNum), []common.Address{{}}, nil) + filter := sys.NewRangeFilter(0, int64(headNum), []common.Address{{}}, nil) filter.Logs(context.Background()) d := time.Since(start) fmt.Println("Finished running filter benchmarks") diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 7767deb0ed..65d25726fa 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -18,6 +18,7 @@ package filters import ( "context" + "errors" "math/big" "github.com/tomochain/tomochain/common" @@ -34,14 +35,19 @@ type Backend interface { ChainDb() ethdb.Database EventMux() *event.TypeMux HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) + HeaderByHash(ctx context.Context, blockHash common.Hash) (*types.Header, error) + GetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) - GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) - ChainConfig() *params.ChainConfig + GetLogs(ctx context.Context, hash common.Hash, number uint64) ([][]*types.Log, error) + PendingBlockAndReceipts() (*types.Block, types.Receipts) + CurrentHeader() *types.Header + ChainConfig() *params.ChainConfig SubscribeTxPreEvent(chan<- core.TxPreEvent) event.Subscription SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription + SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription BloomStatus() (uint64, uint64) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) @@ -49,19 +55,20 @@ type Backend interface { // Filter can be used to retrieve and filter logs. type Filter struct { - backend Backend + sys *FilterSystem + + addresses []common.Address + topics [][]common.Hash - db ethdb.Database - begin, end int64 - addresses []common.Address - topics [][]common.Hash + block *common.Hash // Block hash if filtering a single block + begin, end int64 // Range interval if filtering multiple blocks matcher *bloombits.Matcher } -// New creates a new filter which uses a bloom filter on blocks to figure out whether -// a particular block is interesting or not. -func New(backend Backend, begin, end int64, addresses []common.Address, topics [][]common.Hash) *Filter { +// NewRangeFilter creates a new filter which uses a bloom filter on blocks to +// figure out whether a particular block is interesting or not. +func (sys *FilterSystem) NewRangeFilter(begin, end int64, addresses []common.Address, topics [][]common.Hash) *Filter { // Flatten the address and topic filter clauses into a single bloombits filter // system. Since the bloombits are not positional, nil topics are permitted, // which get flattened into a nil byte slice. @@ -80,74 +87,177 @@ func New(backend Backend, begin, end int64, addresses []common.Address, topics [ } filters = append(filters, filter) } - // Assemble and return the filter - size, _ := backend.BloomStatus() + size, _ := sys.backend.BloomStatus() + // Create a generic filter and convert it into a range filter + filter := newFilter(sys, addresses, topics) + + filter.matcher = bloombits.NewMatcher(size, filters) + filter.begin = begin + filter.end = end + + return filter +} + +// NewBlockFilter creates a new filter which directly inspects the contents of +// a block to figure out whether it is interesting or not. +func (sys *FilterSystem) NewBlockFilter(block common.Hash, addresses []common.Address, topics [][]common.Hash) *Filter { + // Create a generic filter and convert it into a block filter + filter := newFilter(sys, addresses, topics) + filter.block = &block + return filter +} + +// newFilter creates a generic filter that can either filter based on a block hash, +// or based on range queries. The search criteria needs to be explicitly set. +func newFilter(sys *FilterSystem, addresses []common.Address, topics [][]common.Hash) *Filter { return &Filter{ - backend: backend, - begin: begin, - end: end, + sys: sys, addresses: addresses, topics: topics, - db: backend.ChainDb(), - matcher: bloombits.NewMatcher(size, filters), } } // Logs searches the blockchain for matching log entries, returning all from the // first block that contains matches, updating the start of the filter accordingly. func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { - // Figure out the limits of the filter range - header, _ := f.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber) - if header == nil { - return nil, nil + // If we're doing singleton block filtering, execute and return + if f.block != nil { + header, err := f.sys.backend.HeaderByHash(ctx, *f.block) + if err != nil { + return nil, err + } + if header == nil { + return nil, errors.New("unknown block") + } + return f.blockLogs(ctx, header) } - head := header.Number.Uint64() - if f.begin == -1 { - f.begin = int64(head) + var ( + beginPending = f.begin == rpc.PendingBlockNumber.Int64() + endPending = f.end == rpc.PendingBlockNumber.Int64() + ) + + // special case for pending logs + if beginPending && !endPending { + return nil, errors.New("invalid block range") } - end := uint64(f.end) - if f.end == -1 { - end = head + + // Short-cut if all we care about is pending logs + if beginPending && endPending { + return f.pendingLogs(), nil } - // Gather all indexed logs, and finish with non indexed ones + + resolveSpecial := func(number int64) (int64, error) { + var hdr *types.Header + switch number { + case rpc.LatestBlockNumber.Int64(), rpc.PendingBlockNumber.Int64(): + // we should return head here since we've already captured + // that we need to get the pending logs in the pending boolean above + hdr, _ = f.sys.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber) + if hdr == nil { + return 0, errors.New("latest header not found") + } + case rpc.FinalizedBlockNumber.Int64(): + hdr, _ = f.sys.backend.HeaderByNumber(ctx, rpc.FinalizedBlockNumber) + if hdr == nil { + return 0, errors.New("finalized header not found") + } + case rpc.SafeBlockNumber.Int64(): + hdr, _ = f.sys.backend.HeaderByNumber(ctx, rpc.SafeBlockNumber) + if hdr == nil { + return 0, errors.New("safe header not found") + } + default: + return number, nil + } + return hdr.Number.Int64(), nil + } + + var err error + // range query need to resolve the special begin/end block number + if f.begin, err = resolveSpecial(f.begin); err != nil { + return nil, err + } + if f.end, err = resolveSpecial(f.end); err != nil { + return nil, err + } + + logChan, errChan := f.rangeLogsAsync(ctx) + var logs []*types.Log + for { + select { + case log := <-logChan: + logs = append(logs, log) + case err := <-errChan: + if err != nil { + // if an error occurs during extraction, we do return the extracted data + return logs, err + } + // Append the pending ones + if endPending { + pendingLogs := f.pendingLogs() + logs = append(logs, pendingLogs...) + } + return logs, nil + } + } +} + +// rangeLogsAsync retrieves block-range logs that match the filter criteria asynchronously, +// it creates and returns two channels: one for delivering log data, and one for reporting errors. +func (f *Filter) rangeLogsAsync(ctx context.Context) (chan *types.Log, chan error) { var ( - logs []*types.Log - err error + logChan = make(chan *types.Log) + errChan = make(chan error) ) - size, sections := f.backend.BloomStatus() - if indexed := sections * size; indexed > uint64(f.begin) { - if indexed > end { - logs, err = f.indexedLogs(ctx, end) - } else { - logs, err = f.indexedLogs(ctx, indexed-1) + + go func() { + defer func() { + close(errChan) + close(logChan) + }() + + // Gather all indexed logs, and finish with non indexed ones + var ( + end = uint64(f.end) + size, sections = f.sys.backend.BloomStatus() + err error + ) + if indexed := sections * size; indexed > uint64(f.begin) { + if indexed > end { + indexed = end + 1 + } + if err = f.indexedLogs(ctx, indexed-1, logChan); err != nil { + errChan <- err + return + } } - if err != nil { - return logs, err + + if err := f.unindexedLogs(ctx, end, logChan); err != nil { + errChan <- err + return } - } - rest, err := f.unindexedLogs(ctx, end) - logs = append(logs, rest...) - return logs, err + + errChan <- nil + }() + + return logChan, errChan } // indexedLogs returns the logs matching the filter criteria based on the bloom // bits indexed available locally or via the network. -func (f *Filter) indexedLogs(ctx context.Context, end uint64) ([]*types.Log, error) { +func (f *Filter) indexedLogs(ctx context.Context, end uint64, logChan chan *types.Log) error { // Create a matcher session and request servicing from the backend matches := make(chan uint64, 64) session, err := f.matcher.Start(ctx, uint64(f.begin), end, matches) if err != nil { - return nil, err + return err } defer session.Close() - f.backend.ServiceFilter(ctx, session) - - // Iterate over the matches until exhausted or context closed - var logs []*types.Log + f.sys.backend.ServiceFilter(ctx, session) for { select { @@ -158,77 +268,109 @@ func (f *Filter) indexedLogs(ctx context.Context, end uint64) ([]*types.Log, err if err == nil { f.begin = int64(end) + 1 } - return logs, err + return err } f.begin = int64(number) + 1 // Retrieve the suggested block and pull any truly matching logs - header, err := f.backend.HeaderByNumber(ctx, rpc.BlockNumber(number)) + header, err := f.sys.backend.HeaderByNumber(ctx, rpc.BlockNumber(number)) if header == nil || err != nil { - return logs, err + return err } found, err := f.checkMatches(ctx, header) if err != nil { - return logs, err + return err + } + for _, log := range found { + logChan <- log } - logs = append(logs, found...) case <-ctx.Done(): - return logs, ctx.Err() + return ctx.Err() } } } -// indexedLogs returns the logs matching the filter criteria based on raw block +// unindexedLogs returns the logs matching the filter criteria based on raw block // iteration and bloom matching. -func (f *Filter) unindexedLogs(ctx context.Context, end uint64) ([]*types.Log, error) { - var logs []*types.Log - +func (f *Filter) unindexedLogs(ctx context.Context, end uint64, logChan chan *types.Log) error { for ; f.begin <= int64(end); f.begin++ { - header, err := f.backend.HeaderByNumber(ctx, rpc.BlockNumber(f.begin)) + header, err := f.sys.backend.HeaderByNumber(ctx, rpc.BlockNumber(f.begin)) if header == nil || err != nil { - return logs, err + return err } - if bloomFilter(header.Bloom, f.addresses, f.topics) { - found, err := f.checkMatches(ctx, header) - if err != nil { - return logs, err + found, err := f.blockLogs(ctx, header) + if err != nil { + return err + } + for _, log := range found { + select { + case logChan <- log: + case <-ctx.Done(): + return ctx.Err() } - logs = append(logs, found...) } } - return logs, nil + return nil +} + +// blockLogs returns the logs matching the filter criteria within a single block. +func (f *Filter) blockLogs(ctx context.Context, header *types.Header) ([]*types.Log, error) { + if bloomFilter(header.Bloom, f.addresses, f.topics) { + return f.checkMatches(ctx, header) + } + return nil, nil } // checkMatches checks if the receipts belonging to the given header contain any log events that // match the filter criteria. This function is called when the bloom filter signals a potential match. -func (f *Filter) checkMatches(ctx context.Context, header *types.Header) (logs []*types.Log, err error) { - // Get the logs of the block - logsList, err := f.backend.GetLogs(ctx, header.Hash()) +// skipFilter signals all logs of the given block are requested. +func (f *Filter) checkMatches(ctx context.Context, header *types.Header) ([]*types.Log, error) { + hash := header.Hash() + // Logs in cache are partially filled with context data + // such as tx index, block hash, etc. + // Notably tx hash is NOT filled in because it needs + // access to block body data. + cached, err := f.sys.cachedLogElem(ctx, hash, header.Number.Uint64()) if err != nil { return nil, err } - var unfiltered []*types.Log - for _, logs := range logsList { - unfiltered = append(unfiltered, logs...) + logs := filterLogs(cached.logs, nil, nil, f.addresses, f.topics) + if len(logs) == 0 { + return nil, nil } - logs = filterLogs(unfiltered, nil, nil, f.addresses, f.topics) - if len(logs) > 0 { - // We have matching logs, check if we need to resolve full logs via the light client - if logs[0].TxHash == (common.Hash{}) { - receipts, err := f.backend.GetReceipts(ctx, header.Hash()) - if err != nil { - return nil, err - } - unfiltered = unfiltered[:0] - for _, receipt := range receipts { - unfiltered = append(unfiltered, receipt.Logs...) - } - logs = filterLogs(unfiltered, nil, nil, f.addresses, f.topics) - } + // Most backends will deliver un-derived logs, but check nevertheless. + if len(logs) > 0 && logs[0].TxHash != (common.Hash{}) { return logs, nil } - return nil, nil + + body, err := f.sys.cachedGetBody(ctx, cached, hash, header.Number.Uint64()) + if err != nil { + return nil, err + } + for i, log := range logs { + // Copy log not to modify cache elements + logcopy := *log + logcopy.TxHash = body.Transactions[logcopy.TxIndex].Hash() + logs[i] = &logcopy + } + return logs, nil +} + +// pendingLogs returns the logs matching the filter criteria within the pending block. +func (f *Filter) pendingLogs() []*types.Log { + block, receipts := f.sys.backend.PendingBlockAndReceipts() + if block == nil || receipts == nil { + return nil + } + if bloomFilter(block.Bloom(), f.addresses, f.topics) { + var unfiltered []*types.Log + for _, r := range receipts { + unfiltered = append(unfiltered, r.Logs...) + } + return filterLogs(unfiltered, nil, nil, f.addresses, f.topics) + } + return nil } func includes(addresses []common.Address, a common.Address) bool { diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index 3d92fc1ac7..cfd69245fb 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -23,16 +23,101 @@ import ( "errors" "fmt" "sync" + "sync/atomic" "time" - ethereum "github.com/tomochain/tomochain" + "github.com/tomochain/tomochain" "github.com/tomochain/tomochain/common" + "github.com/tomochain/tomochain/common/lru" "github.com/tomochain/tomochain/core" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/event" "github.com/tomochain/tomochain/rpc" ) +// Config represents the configuration of the filter system. +type Config struct { + LogCacheSize int // maximum number of cached blocks (default: 32) + Timeout time.Duration // how long filters stay active (default: 5min) +} + +func (cfg Config) withDefaults() Config { + if cfg.Timeout == 0 { + cfg.Timeout = 5 * time.Minute + } + if cfg.LogCacheSize == 0 { + cfg.LogCacheSize = 32 + } + return cfg +} + +// FilterSystem holds resources shared by all filters. +type FilterSystem struct { + backend Backend + logsCache *lru.Cache[common.Hash, *logCacheElem] + cfg *Config +} + +// NewFilterSystem creates a filter system. +func NewFilterSystem(backend Backend, config Config) *FilterSystem { + config = config.withDefaults() + return &FilterSystem{ + backend: backend, + logsCache: lru.NewCache[common.Hash, *logCacheElem](config.LogCacheSize), + cfg: &config, + } +} + +type logCacheElem struct { + logs []*types.Log + body atomic.Value +} + +// cachedLogElem loads block logs from the backend and caches the result. +func (sys *FilterSystem) cachedLogElem(ctx context.Context, blockHash common.Hash, number uint64) (*logCacheElem, error) { + cached, ok := sys.logsCache.Get(blockHash) + if ok { + return cached, nil + } + + logs, err := sys.backend.GetLogs(ctx, blockHash, number) + if err != nil { + return nil, err + } + if logs == nil { + return nil, fmt.Errorf("failed to get logs for block #%d (0x%s)", number, blockHash.TerminalString()) + } + // Database logs are un-derived. + // Fill in whatever we can (txHash is inaccessible at this point). + flattened := make([]*types.Log, 0) + var logIdx uint + for i, txLogs := range logs { + for _, log := range txLogs { + log.BlockHash = blockHash + log.BlockNumber = number + log.TxIndex = uint(i) + log.Index = logIdx + logIdx++ + flattened = append(flattened, log) + } + } + elem := &logCacheElem{logs: flattened} + sys.logsCache.Add(blockHash, elem) + return elem, nil +} + +func (sys *FilterSystem) cachedGetBody(ctx context.Context, elem *logCacheElem, hash common.Hash, number uint64) (*types.Body, error) { + if body := elem.body.Load(); body != nil { + return body.(*types.Body), nil + } + body, err := sys.backend.GetBody(ctx, hash, rpc.BlockNumber(number)) + if err != nil { + return nil, err + } + elem.body.Store(body) + return body, nil +} + // Type determines the kind of filter and is used to put the filter in to // the correct bucket when added. type Type byte @@ -76,7 +161,7 @@ type subscription struct { id rpc.ID typ Type created time.Time - logsCrit ethereum.FilterQuery + logsCrit tomochain.FilterQuery logs chan []*types.Log hashes chan common.Hash headers chan *types.Header @@ -87,8 +172,8 @@ type subscription struct { // EventSystem creates subscriptions, processes events and broadcasts them to the // subscription which match the subscription criteria. type EventSystem struct { - mux *event.TypeMux backend Backend + sys *FilterSystem lightMode bool lastHead *types.Header install chan *subscription // install filter for event notification @@ -101,17 +186,16 @@ type EventSystem struct { // // The returned manager has a loop that needs to be stopped with the Stop function // or by stopping the given mux. -func NewEventSystem(mux *event.TypeMux, backend Backend, lightMode bool) *EventSystem { +func NewEventSystem(sys *FilterSystem, lightMode bool) *EventSystem { m := &EventSystem{ - mux: mux, - backend: backend, + sys: sys, + backend: sys.backend, lightMode: lightMode, install: make(chan *subscription), uninstall: make(chan *subscription), } go m.eventLoop() - return m } @@ -163,7 +247,7 @@ func (es *EventSystem) subscribe(sub *subscription) *Subscription { // SubscribeLogs creates a subscription that will write all logs matching the // given criteria to the given logs channel. Default value for the from and to // block is "latest". If the fromBlock > toBlock an error is returned. -func (es *EventSystem) SubscribeLogs(crit ethereum.FilterQuery, logs chan []*types.Log) (*Subscription, error) { +func (es *EventSystem) SubscribeLogs(crit tomochain.FilterQuery, logs chan []*types.Log) (*Subscription, error) { var from, to rpc.BlockNumber if crit.FromBlock == nil { from = rpc.LatestBlockNumber @@ -201,7 +285,7 @@ func (es *EventSystem) SubscribeLogs(crit ethereum.FilterQuery, logs chan []*typ // subscribeMinedPendingLogs creates a subscription that returned mined and // pending logs that match the given criteria. -func (es *EventSystem) subscribeMinedPendingLogs(crit ethereum.FilterQuery, logs chan []*types.Log) *Subscription { +func (es *EventSystem) subscribeMinedPendingLogs(crit tomochain.FilterQuery, logs chan []*types.Log) *Subscription { sub := &subscription{ id: rpc.NewID(), typ: MinedAndPendingLogsSubscription, @@ -218,7 +302,7 @@ func (es *EventSystem) subscribeMinedPendingLogs(crit ethereum.FilterQuery, logs // subscribeLogs creates a subscription that will write all logs matching the // given criteria to the given logs channel. -func (es *EventSystem) subscribeLogs(crit ethereum.FilterQuery, logs chan []*types.Log) *Subscription { +func (es *EventSystem) subscribeLogs(crit tomochain.FilterQuery, logs chan []*types.Log) *Subscription { sub := &subscription{ id: rpc.NewID(), typ: LogsSubscription, @@ -235,7 +319,7 @@ func (es *EventSystem) subscribeLogs(crit ethereum.FilterQuery, logs chan []*typ // subscribePendingLogs creates a subscription that writes transaction hashes for // transactions that enter the transaction pool. -func (es *EventSystem) subscribePendingLogs(crit ethereum.FilterQuery, logs chan []*types.Log) *Subscription { +func (es *EventSystem) subscribePendingLogs(crit tomochain.FilterQuery, logs chan []*types.Log) *Subscription { sub := &subscription{ id: rpc.NewID(), typ: PendingLogsSubscription, @@ -375,7 +459,7 @@ func (es *EventSystem) lightFilterLogs(header *types.Header, addresses []common. // Get the logs of the block ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - logsList, err := es.backend.GetLogs(ctx, header.Hash()) + logsList, err := es.backend.GetLogs(ctx, header.Hash(), header.Number.Uint64()) if err != nil { return nil } @@ -412,8 +496,9 @@ func (es *EventSystem) lightFilterLogs(header *types.Header, addresses []common. // eventLoop (un)installs filters and processes mux events. func (es *EventSystem) eventLoop() { var ( - index = make(filterIndex) - sub = es.mux.Subscribe(core.PendingLogsEvent{}) + index = make(filterIndex) + pendingLogsCh = make(chan []*types.Log, logsChanSize) + pendingLogsSub = es.backend.SubscribePendingLogsEvent(pendingLogsCh) // Subscribe TxPreEvent form txpool txCh = make(chan core.TxPreEvent, txChanSize) txSub = es.backend.SubscribeTxPreEvent(txCh) @@ -429,7 +514,7 @@ func (es *EventSystem) eventLoop() { ) // Unsubscribe all events - defer sub.Unsubscribe() + defer pendingLogsSub.Unsubscribe() defer txSub.Unsubscribe() defer rmLogsSub.Unsubscribe() defer logsSub.Unsubscribe() @@ -441,10 +526,7 @@ func (es *EventSystem) eventLoop() { for { select { - case ev, active := <-sub.Chan(): - if !active { // system stopped - return - } + case ev := <-pendingLogsCh: es.broadcast(index, ev) // Handle subscribed events diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 5aeddde39d..47e538a997 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -18,6 +18,7 @@ package filters import ( "context" + "errors" "fmt" "math/big" "math/rand" @@ -39,19 +40,26 @@ import ( ) type testBackend struct { - mux *event.TypeMux - db ethdb.Database - sections uint64 - txFeed *event.Feed - rmLogsFeed *event.Feed - logsFeed *event.Feed - chainFeed *event.Feed + mux *event.TypeMux + db ethdb.Database + sections uint64 + txFeed *event.Feed + rmLogsFeed *event.Feed + logsFeed *event.Feed + chainFeed *event.Feed + pendingBlock *types.Block + pendingReceipts types.Receipts } func (b *testBackend) ChainConfig() *params.ChainConfig { return params.TestChainConfig } +func (b *testBackend) CurrentHeader() *types.Header { + hdr, _ := b.HeaderByNumber(context.TODO(), rpc.LatestBlockNumber) + return hdr +} + func (b *testBackend) ChainDb() ethdb.Database { return b.db } @@ -78,17 +86,30 @@ func (b *testBackend) GetReceipts(ctx context.Context, blockHash common.Hash) (t return core.GetBlockReceipts(b.db, blockHash, number, b.ChainConfig()), nil } -func (b *testBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) { - number := core.GetBlockNumber(b.db, blockHash) - receipts := core.GetBlockReceipts(b.db, blockHash, number, b.ChainConfig()) +func (b *testBackend) HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) { + number := core.GetBlockNumber(b.db, hash) + if number == core.MissingNumber { + return nil, nil + } + return core.GetHeader(b.db, hash, number), nil +} - logs := make([][]*types.Log, len(receipts)) - for i, receipt := range receipts { - logs[i] = receipt.Logs +func (b *testBackend) GetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error) { + if body := core.GetBody(b.db, hash, uint64(number)); body != nil { + return body, nil } + return nil, errors.New("block body not found") +} + +func (b *testBackend) GetLogs(ctx context.Context, hash common.Hash, number uint64) ([][]*types.Log, error) { + logs := core.ReadLogs(b.db, hash, number, params.TestChainConfig) return logs, nil } +func (b *testBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { + return b.pendingBlock, b.pendingReceipts +} + func (b *testBackend) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription { return b.txFeed.Subscribe(ch) } @@ -105,6 +126,10 @@ func (b *testBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subsc return b.chainFeed.Subscribe(ch) } +func (b *testBackend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { + return b.mux.Subscribe(ch) +} + func (b *testBackend) BloomStatus() (uint64, uint64) { return params.BloomBitsBlocks, b.sections } @@ -136,6 +161,12 @@ func (b *testBackend) ServiceFilter(ctx context.Context, session *bloombits.Matc }() } +func newTestFilterSystem(t testing.TB, db ethdb.Database, cfg Config) (*testBackend, *FilterSystem) { + backend := &testBackend{db: db} + sys := NewFilterSystem(backend, cfg) + return backend, sys +} + // TestBlockSubscription tests if a block subscription returns block hashes for posted chain events. // It creates multiple subscriptions: // - one at the start and should receive all posted chain events and a second (blockHashes) @@ -151,8 +182,9 @@ func TestBlockSubscription(t *testing.T) { rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) chainFeed = new(event.Feed) - backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false) + backend = &testBackend{mux: mux, db: db, txFeed: txFeed, rmLogsFeed: rmLogsFeed, logsFeed: logsFeed, chainFeed: chainFeed} + sys = NewFilterSystem(backend, Config{}) + api = NewPublicFilterAPI(sys, false) g = &core.Genesis{ Config: params.TestChainConfig, BaseFee: big.NewInt(params.InitialBaseFee), @@ -212,8 +244,9 @@ func TestPendingTxFilter(t *testing.T) { rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) chainFeed = new(event.Feed) - backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false) + backend = &testBackend{mux: mux, db: db, txFeed: txFeed, rmLogsFeed: rmLogsFeed, logsFeed: logsFeed, chainFeed: chainFeed} + sys = NewFilterSystem(backend, Config{}) + api = NewPublicFilterAPI(sys, false) transactions = []*types.Transaction{ types.NewTransaction(0, common.HexToAddress("0xb794f5ea0ba39494ce83a213fffba74279579268"), new(big.Int), 0, new(big.Int), nil), @@ -275,8 +308,9 @@ func TestLogFilterCreation(t *testing.T) { rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) chainFeed = new(event.Feed) - backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false) + backend = &testBackend{mux: mux, db: db, txFeed: txFeed, rmLogsFeed: rmLogsFeed, logsFeed: logsFeed, chainFeed: chainFeed} + sys = NewFilterSystem(backend, Config{}) + api = NewPublicFilterAPI(sys, false) testCases = []struct { crit FilterCriteria @@ -324,8 +358,9 @@ func TestInvalidLogFilterCreation(t *testing.T) { rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) chainFeed = new(event.Feed) - backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false) + backend = &testBackend{mux: mux, db: db, txFeed: txFeed, rmLogsFeed: rmLogsFeed, logsFeed: logsFeed, chainFeed: chainFeed} + sys = NewFilterSystem(backend, Config{}) + api = NewPublicFilterAPI(sys, false) ) // different situations where log filter creation should fail. @@ -354,8 +389,9 @@ func TestLogFilter(t *testing.T) { rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) chainFeed = new(event.Feed) - backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false) + backend = &testBackend{mux: mux, db: db, txFeed: txFeed, rmLogsFeed: rmLogsFeed, logsFeed: logsFeed, chainFeed: chainFeed} + sys = NewFilterSystem(backend, Config{}) + api = NewPublicFilterAPI(sys, false) firstAddr = common.HexToAddress("0x1111111111111111111111111111111111111111") secondAddr = common.HexToAddress("0x2222222222222222222222222222222222222222") @@ -473,8 +509,9 @@ func TestPendingLogsSubscription(t *testing.T) { rmLogsFeed = new(event.Feed) logsFeed = new(event.Feed) chainFeed = new(event.Feed) - backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false) + backend = &testBackend{mux: mux, db: db, txFeed: txFeed, rmLogsFeed: rmLogsFeed, logsFeed: logsFeed, chainFeed: chainFeed} + sys = NewFilterSystem(backend, Config{}) + api = NewPublicFilterAPI(sys, false) firstAddr = common.HexToAddress("0x1111111111111111111111111111111111111111") secondAddr = common.HexToAddress("0x2222222222222222222222222222222222222222") diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index 6b74b3e4bd..fb0332b3a9 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -18,11 +18,11 @@ package filters import ( "context" - "io/ioutil" + "encoding/json" "math/big" - "os" "strings" "testing" + "time" "github.com/tomochain/tomochain/accounts/abi" "github.com/tomochain/tomochain/common" @@ -30,9 +30,10 @@ import ( "github.com/tomochain/tomochain/core" "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/types" + "github.com/tomochain/tomochain/core/vm" "github.com/tomochain/tomochain/crypto" - "github.com/tomochain/tomochain/event" "github.com/tomochain/tomochain/params" + "github.com/tomochain/tomochain/rpc" ) func makeReceipt(addr common.Address) *types.Receipt { @@ -45,61 +46,56 @@ func makeReceipt(addr common.Address) *types.Receipt { } func BenchmarkFilters(b *testing.B) { - dir, err := ioutil.TempDir("", "filtertest") - if err != nil { - b.Fatal(err) - } - defer os.RemoveAll(dir) - var ( - db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0, "") - mux = new(event.TypeMux) - txFeed = new(event.Feed) - rmLogsFeed = new(event.Feed) - logsFeed = new(event.Feed) - chainFeed = new(event.Feed) - backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} - key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - addr1 = crypto.PubkeyToAddress(key1.PublicKey) - addr2 = common.BytesToAddress([]byte("jeff")) - addr3 = common.BytesToAddress([]byte("ethereum")) - addr4 = common.BytesToAddress([]byte("random addresses please")) + db, _ = rawdb.NewLevelDBDatabase(b.TempDir(), 0, 0, "") + _, sys = newTestFilterSystem(b, db, Config{}) + key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + addr1 = crypto.PubkeyToAddress(key1.PublicKey) + addr2 = common.BytesToAddress([]byte("jeff")) + addr3 = common.BytesToAddress([]byte("ethereum")) + addr4 = common.BytesToAddress([]byte("random addresses please")) + + gspec = &core.Genesis{ + Alloc: core.GenesisAlloc{addr1: {Balance: big.NewInt(1000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + Config: params.TestChainConfig, + } ) defer db.Close() - - genesis := core.GenesisBlockForTesting(db, addr1, big.NewInt(1000000)) - chain, receipts := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 100010, func(i int, gen *core.BlockGen) { + _, chain, receipts := core.GenerateChainWithGenesis(gspec, ethash.NewFaker(), 100010, func(i int, gen *core.BlockGen) { switch i { case 2403: receipt := makeReceipt(addr1) gen.AddUncheckedReceipt(receipt) + gen.AddUncheckedTx(types.NewTransaction(999, common.HexToAddress("0x999"), big.NewInt(999), 999, gen.BaseFee(), nil)) case 1034: receipt := makeReceipt(addr2) gen.AddUncheckedReceipt(receipt) + gen.AddUncheckedTx(types.NewTransaction(999, common.HexToAddress("0x999"), big.NewInt(999), 999, gen.BaseFee(), nil)) case 34: receipt := makeReceipt(addr3) gen.AddUncheckedReceipt(receipt) + gen.AddUncheckedTx(types.NewTransaction(999, common.HexToAddress("0x999"), big.NewInt(999), 999, gen.BaseFee(), nil)) case 99999: receipt := makeReceipt(addr4) gen.AddUncheckedReceipt(receipt) - + gen.AddUncheckedTx(types.NewTransaction(999, common.HexToAddress("0x999"), big.NewInt(999), 999, gen.BaseFee(), nil)) } }) + // The test txs are not properly signed, can't simply create a chain + // and then import blocks. TODO(rjl493456442) try to get rid of the + // manual database writes. + gspec.MustCommit(db) + for i, block := range chain { core.WriteBlock(db, block) - if err := core.WriteCanonicalHash(db, block.Hash(), block.NumberU64()); err != nil { - b.Fatalf("failed to insert block number: %v", err) - } - if err := core.WriteHeadBlockHash(db, block.Hash()); err != nil { - b.Fatalf("failed to insert block number: %v", err) - } - if err := core.WriteBlockReceipts(db, block.Hash(), block.NumberU64(), receipts[i]); err != nil { - b.Fatal("error writing block receipts:", err) - } + core.WriteCanonicalHash(db, block.Hash(), block.NumberU64()) + core.WriteHeadBlockHash(db, block.Hash()) + core.WriteBlockReceipts(db, block.Hash(), block.NumberU64(), receipts[i]) } b.ResetTimer() - filter := New(backend, 0, -1, []common.Address{addr1, addr2, addr3, addr4}, nil) + filter := sys.NewRangeFilter(0, -1, []common.Address{addr1, addr2, addr3, addr4}, nil) for i := 0; i < b.N; i++ { logs, _ := filter.Logs(context.Background()) @@ -110,41 +106,71 @@ func BenchmarkFilters(b *testing.B) { } func TestFilters(t *testing.T) { - dir, err := ioutil.TempDir("", "filtertest") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - var ( - db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0, "") - mux = new(event.TypeMux) - txFeed = new(event.Feed) - rmLogsFeed = new(event.Feed) - logsFeed = new(event.Feed) - chainFeed = new(event.Feed) - backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} - key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - addr = crypto.PubkeyToAddress(key1.PublicKey) - signer = types.NewLondonSigner(big.NewInt(1)) - + db = rawdb.NewMemoryDatabase() + _, sys = newTestFilterSystem(t, db, Config{}) + // Sender account + key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + addr = crypto.PubkeyToAddress(key1.PublicKey) + signer = types.NewLondonSigner(big.NewInt(1)) // Logging contract contract = common.Address{0xfe} contract2 = common.Address{0xff} abiStr = `[{"inputs":[],"name":"log0","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"uint256","name":"t1","type":"uint256"}],"name":"log1","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"uint256","name":"t1","type":"uint256"},{"internalType":"uint256","name":"t2","type":"uint256"}],"name":"log2","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"uint256","name":"t1","type":"uint256"},{"internalType":"uint256","name":"t2","type":"uint256"},{"internalType":"uint256","name":"t3","type":"uint256"}],"name":"log3","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"uint256","name":"t1","type":"uint256"},{"internalType":"uint256","name":"t2","type":"uint256"},{"internalType":"uint256","name":"t3","type":"uint256"},{"internalType":"uint256","name":"t4","type":"uint256"}],"name":"log4","outputs":[],"stateMutability":"nonpayable","type":"function"}]` + /* + // SPDX-License-Identifier: GPL-3.0 + pragma solidity >=0.7.0 <0.9.0; + + contract Logger { + function log0() external { + assembly { + log0(0, 0) + } + } + + function log1(uint t1) external { + assembly { + log1(0, 0, t1) + } + } + + function log2(uint t1, uint t2) external { + assembly { + log2(0, 0, t1, t2) + } + } + + function log3(uint t1, uint t2, uint t3) external { + assembly { + log3(0, 0, t1, t2, t3) + } + } + + function log4(uint t1, uint t2, uint t3, uint t4) external { + assembly { + log4(0, 0, t1, t2, t3, t4) + } + } + } + */ + bytecode = common.FromHex("608060405234801561001057600080fd5b50600436106100575760003560e01c80630aa731851461005c5780632a4c08961461006657806378b9a1f314610082578063c670f8641461009e578063c683d6a3146100ba575b600080fd5b6100646100d6565b005b610080600480360381019061007b9190610143565b6100dc565b005b61009c60048036038101906100979190610196565b6100e8565b005b6100b860048036038101906100b391906101d6565b6100f2565b005b6100d460048036038101906100cf9190610203565b6100fa565b005b600080a0565b808284600080a3505050565b8082600080a25050565b80600080a150565b80828486600080a450505050565b600080fd5b6000819050919050565b6101208161010d565b811461012b57600080fd5b50565b60008135905061013d81610117565b92915050565b60008060006060848603121561015c5761015b610108565b5b600061016a8682870161012e565b935050602061017b8682870161012e565b925050604061018c8682870161012e565b9150509250925092565b600080604083850312156101ad576101ac610108565b5b60006101bb8582860161012e565b92505060206101cc8582860161012e565b9150509250929050565b6000602082840312156101ec576101eb610108565b5b60006101fa8482850161012e565b91505092915050565b6000806000806080858703121561021d5761021c610108565b5b600061022b8782880161012e565b945050602061023c8782880161012e565b935050604061024d8782880161012e565b925050606061025e8782880161012e565b9150509295919450925056fea264697066735822122073a4b156f487e59970dc1ef449cc0d51467268f676033a17188edafcee861f9864736f6c63430008110033") hash1 = common.BytesToHash([]byte("topic1")) hash2 = common.BytesToHash([]byte("topic2")) hash3 = common.BytesToHash([]byte("topic3")) hash4 = common.BytesToHash([]byte("topic4")) - ) - defer db.Close() + hash5 = common.BytesToHash([]byte("topic5")) - g := &core.Genesis{ - Config: params.TestChainConfig, - BaseFee: big.NewInt(params.InitialBaseFee), - Alloc: core.GenesisAlloc{addr: {Balance: big.NewInt(0).Mul(big.NewInt(100), big.NewInt(params.Ether))}}, - } + gspec = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{ + addr: {Balance: big.NewInt(0).Mul(big.NewInt(100), big.NewInt(params.Ether))}, + contract: {Balance: big.NewInt(0), Code: bytecode}, + contract2: {Balance: big.NewInt(0), Code: bytecode}, + }, + BaseFee: big.NewInt(params.InitialBaseFee), + } + ) contractABI, err := abi.JSON(strings.NewReader(abiStr)) if err != nil { @@ -153,11 +179,11 @@ func TestFilters(t *testing.T) { // Hack: GenerateChainWithGenesis creates a new db. // Commit the genesis manually and use GenerateChain. - _, err = g.Commit(db) + _, err = gspec.Commit(db) if err != nil { t.Fatal(err) } - chain, receipts := core.GenerateChain(g.Config, g.ToBlock(db), ethash.NewFaker(), db, 1000, func(i int, gen *core.BlockGen) { + chain, _ := core.GenerateChain(gspec.Config, gspec.ToBlock(db), ethash.NewFaker(), db, 1000, func(i int, gen *core.BlockGen) { switch i { case 1: data, err := contractABI.Pack("log1", hash1.Big()) @@ -221,71 +247,120 @@ func TestFilters(t *testing.T) { gen.AddTx(tx) } }) - for i, block := range chain { - core.WriteBlock(db, block) - if err := core.WriteCanonicalHash(db, block.Hash(), block.NumberU64()); err != nil { - t.Fatalf("failed to insert block number: %v", err) - } - if err := core.WriteHeadBlockHash(db, block.Hash()); err != nil { - t.Fatalf("failed to insert block number: %v", err) - } - if err := core.WriteBlockReceipts(db, block.Hash(), block.NumberU64(), receipts[i]); err != nil { - t.Fatal("error writing block receipts:", err) - } - } - - filter := New(backend, 0, -1, []common.Address{addr}, [][]common.Hash{{hash1, hash2, hash3, hash4}}) - - logs, _ := filter.Logs(context.Background()) - if len(logs) != 4 { - t.Error("expected 4 log, got", len(logs)) - } - - filter = New(backend, 900, 999, []common.Address{addr}, [][]common.Hash{{hash3}}) - logs, _ = filter.Logs(context.Background()) - if len(logs) != 1 { - t.Error("expected 1 log, got", len(logs)) - } - if len(logs) > 0 && logs[0].Topics[0] != hash3 { - t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0]) - } - - filter = New(backend, 990, -1, []common.Address{addr}, [][]common.Hash{{hash3}}) - logs, _ = filter.Logs(context.Background()) - if len(logs) != 1 { - t.Error("expected 1 log, got", len(logs)) - } - if len(logs) > 0 && logs[0].Topics[0] != hash3 { - t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0]) - } - - filter = New(backend, 1, 10, nil, [][]common.Hash{{hash1, hash2}}) - - logs, _ = filter.Logs(context.Background()) - if len(logs) != 2 { - t.Error("expected 2 log, got", len(logs)) + bc, err := core.NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{NoBaseFee: true}) + if err != nil { + t.Fatal(err) } - - failHash := common.BytesToHash([]byte("fail")) - filter = New(backend, 0, -1, nil, [][]common.Hash{{failHash}}) - - logs, _ = filter.Logs(context.Background()) - if len(logs) != 0 { - t.Error("expected 0 log, got", len(logs)) + _, err = bc.InsertChain(chain) + if err != nil { + t.Fatal(err) } - failAddr := common.BytesToAddress([]byte("failmenow")) - filter = New(backend, 0, -1, []common.Address{failAddr}, nil) + // Generate pending block + pchain, preceipts := core.GenerateChain(gspec.Config, chain[len(chain)-1], ethash.NewFaker(), db, 1, func(i int, gen *core.BlockGen) { + data, err := contractABI.Pack("log1", hash5.Big()) + if err != nil { + t.Fatal(err) + } + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 5, + GasPrice: gen.BaseFee(), + Gas: 30000, + To: &contract, + Data: data, + }), signer, key1) + gen.AddTx(tx) + }) + sys.backend.(*testBackend).pendingBlock = pchain[0] + sys.backend.(*testBackend).pendingReceipts = preceipts[0] - logs, _ = filter.Logs(context.Background()) - if len(logs) != 0 { - t.Error("expected 0 log, got", len(logs)) + for i, tc := range []struct { + f *Filter + want string + err string + }{ + { + f: sys.NewBlockFilter(chain[2].Hash(), []common.Address{contract}, nil), + want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0xdefe471992a07a02acdfbe33edaae22fbb86d7d3cec3f1b8e4e77702fb3acc1d","transactionIndex":"0x0","blockHash":"0x7a7556792ca7d37882882e2b001fe14833eaf81c2c7f865c9c771ec37a024f6b","logIndex":"0x0","removed":false}]`, + }, { + f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), []common.Address{contract}, [][]common.Hash{{hash1, hash2, hash3, hash4}}), + want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x2","transactionHash":"0xa8028c655b6423204c8edfbc339f57b042d6bec2b6a61145d76b7c08b4cccd42","transactionIndex":"0x0","blockHash":"0x24417bb49ce44cfad65da68f33b510bf2a129c0d89ccf06acb6958b8585ccf34","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0xdefe471992a07a02acdfbe33edaae22fbb86d7d3cec3f1b8e4e77702fb3acc1d","transactionIndex":"0x0","blockHash":"0x7a7556792ca7d37882882e2b001fe14833eaf81c2c7f865c9c771ec37a024f6b","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0xb360bad5265261c075ece02d3bf0e39498a6a76310482cdfd90588748e6c5ee0","logIndex":"0x0","removed":false}]`, + }, { + f: sys.NewRangeFilter(900, 999, []common.Address{contract}, [][]common.Hash{{hash3}}), + }, { + f: sys.NewRangeFilter(990, int64(rpc.LatestBlockNumber), []common.Address{contract2}, [][]common.Hash{{hash3}}), + want: `[{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696333"],"data":"0x","blockNumber":"0x3e7","transactionHash":"0x53e3675800c6908424b61b35a44e51ca4c73ca603e58a65b32c67968b4f42200","transactionIndex":"0x0","blockHash":"0x2e4620a2b426b0612ec6cad9603f466723edaed87f98c9137405dd4f7a2409ff","logIndex":"0x0","removed":false}]`, + }, { + f: sys.NewRangeFilter(1, 10, []common.Address{contract}, [][]common.Hash{{hash2}, {hash1}}), + want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0xdefe471992a07a02acdfbe33edaae22fbb86d7d3cec3f1b8e4e77702fb3acc1d","transactionIndex":"0x0","blockHash":"0x7a7556792ca7d37882882e2b001fe14833eaf81c2c7f865c9c771ec37a024f6b","logIndex":"0x0","removed":false}]`, + }, { + f: sys.NewRangeFilter(1, 10, nil, [][]common.Hash{{hash1, hash2}}), + want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x2","transactionHash":"0xa8028c655b6423204c8edfbc339f57b042d6bec2b6a61145d76b7c08b4cccd42","transactionIndex":"0x0","blockHash":"0x24417bb49ce44cfad65da68f33b510bf2a129c0d89ccf06acb6958b8585ccf34","logIndex":"0x0","removed":false},{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x2","transactionHash":"0xdba3e2ea9a7d690b722d70ee605fd67ba4c00d1d3aecd5cf187a7b92ad8eb3df","transactionIndex":"0x1","blockHash":"0x24417bb49ce44cfad65da68f33b510bf2a129c0d89ccf06acb6958b8585ccf34","logIndex":"0x1","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0xdefe471992a07a02acdfbe33edaae22fbb86d7d3cec3f1b8e4e77702fb3acc1d","transactionIndex":"0x0","blockHash":"0x7a7556792ca7d37882882e2b001fe14833eaf81c2c7f865c9c771ec37a024f6b","logIndex":"0x0","removed":false}]`, + }, { + f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), nil, [][]common.Hash{{common.BytesToHash([]byte("fail"))}}), + }, { + f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), []common.Address{common.BytesToAddress([]byte("failmenow"))}, nil), + }, { + f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), nil, [][]common.Hash{{common.BytesToHash([]byte("fail"))}, {hash1}}), + }, { + f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.LatestBlockNumber), nil, nil), + want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0xb360bad5265261c075ece02d3bf0e39498a6a76310482cdfd90588748e6c5ee0","logIndex":"0x0","removed":false}]`, + }, { + f: sys.NewRangeFilter(int64(rpc.FinalizedBlockNumber), int64(rpc.LatestBlockNumber), nil, nil), + want: `[{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696333"],"data":"0x","blockNumber":"0x3e7","transactionHash":"0x53e3675800c6908424b61b35a44e51ca4c73ca603e58a65b32c67968b4f42200","transactionIndex":"0x0","blockHash":"0x2e4620a2b426b0612ec6cad9603f466723edaed87f98c9137405dd4f7a2409ff","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0xb360bad5265261c075ece02d3bf0e39498a6a76310482cdfd90588748e6c5ee0","logIndex":"0x0","removed":false}]`, + }, { + f: sys.NewRangeFilter(int64(rpc.FinalizedBlockNumber), int64(rpc.FinalizedBlockNumber), nil, nil), + want: `[{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696333"],"data":"0x","blockNumber":"0x3e7","transactionHash":"0x53e3675800c6908424b61b35a44e51ca4c73ca603e58a65b32c67968b4f42200","transactionIndex":"0x0","blockHash":"0x2e4620a2b426b0612ec6cad9603f466723edaed87f98c9137405dd4f7a2409ff","logIndex":"0x0","removed":false}]`, + }, { + f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.FinalizedBlockNumber), nil, nil), + }, { + f: sys.NewRangeFilter(int64(rpc.SafeBlockNumber), int64(rpc.LatestBlockNumber), nil, nil), + err: "safe header not found", + }, { + f: sys.NewRangeFilter(int64(rpc.SafeBlockNumber), int64(rpc.SafeBlockNumber), nil, nil), + err: "safe header not found", + }, { + f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.SafeBlockNumber), nil, nil), + err: "safe header not found", + }, { + f: sys.NewRangeFilter(int64(rpc.PendingBlockNumber), int64(rpc.PendingBlockNumber), nil, nil), + want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696335"],"data":"0x","blockNumber":"0x3e9","transactionHash":"0x4110587c1b8d86edc85dce929a34127f1cb8809515a9f177c91c866de3eb0638","transactionIndex":"0x0","blockHash":"0xc7245899e5817f16fa99cf5ad2d9c1e4b98443a565a673ec9c764640443ef037","logIndex":"0x0","removed":false}]`, + }, { + f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.PendingBlockNumber), nil, nil), + want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0xb360bad5265261c075ece02d3bf0e39498a6a76310482cdfd90588748e6c5ee0","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696335"],"data":"0x","blockNumber":"0x3e9","transactionHash":"0x4110587c1b8d86edc85dce929a34127f1cb8809515a9f177c91c866de3eb0638","transactionIndex":"0x0","blockHash":"0xc7245899e5817f16fa99cf5ad2d9c1e4b98443a565a673ec9c764640443ef037","logIndex":"0x0","removed":false}]`, + }, { + f: sys.NewRangeFilter(int64(rpc.PendingBlockNumber), int64(rpc.LatestBlockNumber), nil, nil), + err: "invalid block range", + }, + } { + logs, err := tc.f.Logs(context.Background()) + if err == nil && tc.err != "" { + t.Fatalf("test %d, expected error %q, got nil", i, tc.err) + } else if err != nil && err.Error() != tc.err { + t.Fatalf("test %d, expected error %q, got %q", i, tc.err, err.Error()) + } + if tc.want == "" && len(logs) == 0 { + continue + } + have, err := json.Marshal(logs) + if err != nil { + t.Fatal(err) + } + if string(have) != tc.want { + t.Fatalf("test %d, have:\n%s\nwant:\n%s", i, have, tc.want) + } } - filter = New(backend, 0, -1, nil, [][]common.Hash{{failHash}, {hash1}}) - - logs, _ = filter.Logs(context.Background()) - if len(logs) != 0 { - t.Error("expected 0 log, got", len(logs)) - } + t.Run("timeout", func(t *testing.T) { + f := sys.NewRangeFilter(0, -1, nil, nil) + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Hour)) + defer cancel() + _, err := f.Logs(ctx) + if err == nil { + t.Fatal("expected error") + } + if err != context.DeadlineExceeded { + t.Fatalf("expected context.DeadlineExceeded, got %v", err) + } + }) } diff --git a/event/event.go b/event/event.go index 20d20d1f57..dae455225d 100644 --- a/event/event.go +++ b/event/event.go @@ -158,6 +158,8 @@ type TypeMuxSubscription struct { postMu sync.RWMutex readC <-chan *TypeMuxEvent postC chan<- *TypeMuxEvent + + err chan error } func newsub(mux *TypeMux) *TypeMuxSubscription { @@ -209,3 +211,8 @@ func (s *TypeMuxSubscription) deliver(event *TypeMuxEvent) { case <-s.closing: } } + +// Err returns a channel that is closed when unsubscribed. +func (s *TypeMuxSubscription) Err() <-chan error { + return s.err +} diff --git a/interfaces.go b/interfaces.go index b19adda036..c8da84cab8 100644 --- a/interfaces.go +++ b/interfaces.go @@ -136,6 +136,7 @@ type ContractCaller interface { // FilterQuery contains options for contract log filtering. type FilterQuery struct { + BlockHash *common.Hash // used by eth_getLogs, return logs only from block with this hash FromBlock *big.Int // beginning of the queried range, nil means genesis block ToBlock *big.Int // end of the range, nil means latest block Addresses []common.Address // restricts matches to events created by specific contracts diff --git a/les/api_backend.go b/les/api_backend.go index 15a3ebe8e2..cafeb5858f 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -105,6 +105,14 @@ func (b *LesApiBackend) BlockByNumber(ctx context.Context, blockNr rpc.BlockNumb return b.GetBlock(ctx, header.Hash()) } +func (b *LesApiBackend) GetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error) { + return light.GetBody(ctx, b.eth.odr, hash, uint64(number)) +} + +func (b *LesApiBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { + return nil, nil +} + func (b *LesApiBackend) StateAndHeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*state.StateDB, *types.Header, error) { header, err := b.HeaderByNumber(ctx, blockNr) if header == nil || err != nil { @@ -138,7 +146,7 @@ func (b *LesApiBackend) GetReceipts(ctx context.Context, blockHash common.Hash) return light.GetBlockReceipts(ctx, b.eth.odr, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash), b.ChainConfig()) } -func (b *LesApiBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) { +func (b *LesApiBackend) GetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error) { return light.GetBlockLogs(ctx, b.eth.odr, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash), b.ChainConfig()) } @@ -217,6 +225,13 @@ func (b *LesApiBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEven return b.eth.blockchain.SubscribeRemovedLogsEvent(ch) } +func (b *LesApiBackend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { + return event.NewSubscription(func(quit <-chan struct{}) error { + <-quit + return nil + }) +} + func (b *LesApiBackend) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { return b.gpo.SuggestTipCap(ctx) } diff --git a/les/backend.go b/les/backend.go index 1a5cae11b8..0a6dab6f7b 100644 --- a/les/backend.go +++ b/les/backend.go @@ -176,6 +176,9 @@ func (s *LightDummyAPI) Mining() bool { // APIs returns the collection of RPC services the ethereum package offers. // NOTE, some of these services probably need to be moved to somewhere else. func (s *LightEthereum) APIs() []rpc.API { + filterSystem := filters.NewFilterSystem(s.ApiBackend, filters.Config{ + LogCacheSize: s.config.FilterLogCacheSize, + }) return append(ethapi.GetAPIs(s.ApiBackend), []rpc.API{ { Namespace: "eth", @@ -190,7 +193,7 @@ func (s *LightEthereum) APIs() []rpc.API { }, { Namespace: "eth", Version: "1.0", - Service: filters.NewPublicFilterAPI(s.ApiBackend, true), + Service: filters.NewPublicFilterAPI(filterSystem, true), Public: true, }, { Namespace: "net", diff --git a/miner/miner.go b/miner/miner.go index 1d40cd5a70..570ec487fe 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -19,7 +19,6 @@ package miner import ( "fmt" - "github.com/tomochain/tomochain/tomoxlending" "sync/atomic" "github.com/tomochain/tomochain/accounts" @@ -34,6 +33,7 @@ import ( "github.com/tomochain/tomochain/log" "github.com/tomochain/tomochain/params" "github.com/tomochain/tomochain/tomox" + "github.com/tomochain/tomochain/tomoxlending" ) // Backend wraps all methods required for mining. @@ -177,7 +177,19 @@ func (self *Miner) PendingBlock() *types.Block { return self.worker.pendingBlock() } +// PendingBlockAndReceipts returns the currently pending block and corresponding receipts. +// The returned values can be nil in case the pending block is not initialized. +func (self *Miner) PendingBlockAndReceipts() (*types.Block, types.Receipts) { + return self.worker.pendingBlockAndReceipts() +} + func (self *Miner) SetEtherbase(addr common.Address) { self.coinbase = addr self.worker.setEtherbase(addr) } + +// SubscribePendingLogs starts delivering logs from pending transactions +// to the given channel. +func (self *Miner) SubscribePendingLogs(ch chan<- []*types.Log) event.Subscription { + return self.worker.mux.Subscribe(ch) +} diff --git a/miner/worker.go b/miner/worker.go index 3d9b75189b..d2b0c991d8 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -224,6 +224,14 @@ func (self *worker) pendingBlock() *types.Block { return self.current.Block } +// pendingBlockAndReceipts returns pending block and corresponding receipts. +// The returned values can be nil in case the pending block is not initialized. +func (self *worker) pendingBlockAndReceipts() (*types.Block, types.Receipts) { + self.currentMu.Lock() + defer self.currentMu.Unlock() + return self.current.Block, self.current.receipts +} + func (self *worker) start() { self.mu.Lock() defer self.mu.Unlock() From e0749cecea8b529f85b6e00808a33a3ab668c4da Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 12 Jul 2023 21:33:32 +0700 Subject: [PATCH 76/82] Fix filter unit tests --- eth/filters/filter_system_test.go | 28 +++++++++++++++------------- eth/filters/filter_test.go | 24 ++++++++---------------- 2 files changed, 23 insertions(+), 29 deletions(-) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 47e538a997..c97dee2c86 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -410,8 +410,8 @@ func TestLogFilter(t *testing.T) { {Address: thirdAddress, Topics: []common.Hash{secondTopic}, BlockNumber: 3}, } - expectedCase7 = []*types.Log{allLogs[3], allLogs[4], allLogs[0], allLogs[1], allLogs[2], allLogs[3], allLogs[4]} - expectedCase11 = []*types.Log{allLogs[1], allLogs[2], allLogs[1], allLogs[2]} + //expectedCase7 = []*types.Log{allLogs[3], allLogs[4], allLogs[0], allLogs[1], allLogs[2], allLogs[3], allLogs[4]} + //expectedCase11 = []*types.Log{allLogs[1], allLogs[2], allLogs[1], allLogs[2]} testCases = []struct { crit FilterCriteria @@ -430,20 +430,22 @@ func TestLogFilter(t *testing.T) { 4: {FilterCriteria{Addresses: []common.Address{thirdAddress}, Topics: [][]common.Hash{{firstTopic, secondTopic}}}, allLogs[3:5], ""}, // match logs based on multiple addresses and "or" topics 5: {FilterCriteria{Addresses: []common.Address{secondAddr, thirdAddress}, Topics: [][]common.Hash{{firstTopic, secondTopic}}}, allLogs[2:5], ""}, - // logs in the pending block - 6: {FilterCriteria{Addresses: []common.Address{firstAddr}, FromBlock: big.NewInt(rpc.PendingBlockNumber.Int64()), ToBlock: big.NewInt(rpc.PendingBlockNumber.Int64())}, allLogs[:2], ""}, - // mined logs with block num >= 2 or pending logs - 7: {FilterCriteria{FromBlock: big.NewInt(2), ToBlock: big.NewInt(rpc.PendingBlockNumber.Int64())}, expectedCase7, ""}, // all "mined" logs with block num >= 2 - 8: {FilterCriteria{FromBlock: big.NewInt(2), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())}, allLogs[3:], ""}, + 6: {FilterCriteria{FromBlock: big.NewInt(2), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())}, allLogs[3:], ""}, // all "mined" logs - 9: {FilterCriteria{ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())}, allLogs, ""}, + 7: {FilterCriteria{ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())}, allLogs, ""}, // all "mined" logs with 1>= block num <=2 and topic secondTopic - 10: {FilterCriteria{FromBlock: big.NewInt(1), ToBlock: big.NewInt(2), Topics: [][]common.Hash{{secondTopic}}}, allLogs[3:4], ""}, - // all "mined" and pending logs with topic firstTopic - 11: {FilterCriteria{FromBlock: big.NewInt(rpc.LatestBlockNumber.Int64()), ToBlock: big.NewInt(rpc.PendingBlockNumber.Int64()), Topics: [][]common.Hash{{firstTopic}}}, expectedCase11, ""}, + 8: {FilterCriteria{FromBlock: big.NewInt(1), ToBlock: big.NewInt(2), Topics: [][]common.Hash{{secondTopic}}}, allLogs[3:4], ""}, // match all logs due to wildcard topic - 12: {FilterCriteria{Topics: [][]common.Hash{nil}}, allLogs[1:], ""}, + 9: {FilterCriteria{Topics: [][]common.Hash{nil}}, allLogs[1:], ""}, + /* + // logs in the pending block + 10: {FilterCriteria{Addresses: []common.Address{firstAddr}, FromBlock: big.NewInt(rpc.PendingBlockNumber.Int64()), ToBlock: big.NewInt(rpc.PendingBlockNumber.Int64())}, allLogs[:2], ""}, + // mined logs with block num >= 2 or pending logs + 11: {FilterCriteria{FromBlock: big.NewInt(2), ToBlock: big.NewInt(rpc.PendingBlockNumber.Int64())}, expectedCase7, ""}, + // all "mined" and pending logs with topic firstTopic + 12: {FilterCriteria{FromBlock: big.NewInt(rpc.LatestBlockNumber.Int64()), ToBlock: big.NewInt(rpc.PendingBlockNumber.Int64()), Topics: [][]common.Hash{{firstTopic}}}, expectedCase11, ""}, + */ } ) @@ -484,7 +486,7 @@ func TestLogFilter(t *testing.T) { if len(fetched) != len(tt.expected) { t.Errorf("invalid number of logs for case %d, want %d log(s), got %d", i, len(tt.expected), len(fetched)) - return + //return } for l := range fetched { diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index fb0332b3a9..45fdc8ebfe 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -281,21 +281,21 @@ func TestFilters(t *testing.T) { }{ { f: sys.NewBlockFilter(chain[2].Hash(), []common.Address{contract}, nil), - want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0xdefe471992a07a02acdfbe33edaae22fbb86d7d3cec3f1b8e4e77702fb3acc1d","transactionIndex":"0x0","blockHash":"0x7a7556792ca7d37882882e2b001fe14833eaf81c2c7f865c9c771ec37a024f6b","logIndex":"0x0","removed":false}]`, + want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0x8347f069b3cefebee185fa93df3086c284f8a2b2088d9b7030c970e8d48758b5","transactionIndex":"0x0","blockHash":"0xe5e2e1696398a43b3a464afcfee2f46271ee90800527a4abe50155a7666e5e7c","logIndex":"0x0","removed":false}]`, }, { f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), []common.Address{contract}, [][]common.Hash{{hash1, hash2, hash3, hash4}}), - want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x2","transactionHash":"0xa8028c655b6423204c8edfbc339f57b042d6bec2b6a61145d76b7c08b4cccd42","transactionIndex":"0x0","blockHash":"0x24417bb49ce44cfad65da68f33b510bf2a129c0d89ccf06acb6958b8585ccf34","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0xdefe471992a07a02acdfbe33edaae22fbb86d7d3cec3f1b8e4e77702fb3acc1d","transactionIndex":"0x0","blockHash":"0x7a7556792ca7d37882882e2b001fe14833eaf81c2c7f865c9c771ec37a024f6b","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0xb360bad5265261c075ece02d3bf0e39498a6a76310482cdfd90588748e6c5ee0","logIndex":"0x0","removed":false}]`, + want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x2","transactionHash":"0xa8028c655b6423204c8edfbc339f57b042d6bec2b6a61145d76b7c08b4cccd42","transactionIndex":"0x0","blockHash":"0x050509686626eedc759e82b52ee87a22d760e1242b2ab81d6b95e24f4d12cd97","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0x8347f069b3cefebee185fa93df3086c284f8a2b2088d9b7030c970e8d48758b5","transactionIndex":"0x0","blockHash":"0xe5e2e1696398a43b3a464afcfee2f46271ee90800527a4abe50155a7666e5e7c","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0x4d2c709f2f80120b55f5fa8d9a55d15abadbefceaf00102ed942db8ebfbc0cfb","logIndex":"0x0","removed":false}]`, }, { f: sys.NewRangeFilter(900, 999, []common.Address{contract}, [][]common.Hash{{hash3}}), }, { f: sys.NewRangeFilter(990, int64(rpc.LatestBlockNumber), []common.Address{contract2}, [][]common.Hash{{hash3}}), - want: `[{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696333"],"data":"0x","blockNumber":"0x3e7","transactionHash":"0x53e3675800c6908424b61b35a44e51ca4c73ca603e58a65b32c67968b4f42200","transactionIndex":"0x0","blockHash":"0x2e4620a2b426b0612ec6cad9603f466723edaed87f98c9137405dd4f7a2409ff","logIndex":"0x0","removed":false}]`, + want: `[{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696333"],"data":"0x","blockNumber":"0x3e7","transactionHash":"0x53e3675800c6908424b61b35a44e51ca4c73ca603e58a65b32c67968b4f42200","transactionIndex":"0x0","blockHash":"0x6ed20c797ca398e48e71b7b09618f40d7a484371e0f9ae007ca504f0cb080035","logIndex":"0x0","removed":false}]`, }, { f: sys.NewRangeFilter(1, 10, []common.Address{contract}, [][]common.Hash{{hash2}, {hash1}}), - want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0xdefe471992a07a02acdfbe33edaae22fbb86d7d3cec3f1b8e4e77702fb3acc1d","transactionIndex":"0x0","blockHash":"0x7a7556792ca7d37882882e2b001fe14833eaf81c2c7f865c9c771ec37a024f6b","logIndex":"0x0","removed":false}]`, + want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0x8347f069b3cefebee185fa93df3086c284f8a2b2088d9b7030c970e8d48758b5","transactionIndex":"0x0","blockHash":"0xe5e2e1696398a43b3a464afcfee2f46271ee90800527a4abe50155a7666e5e7c","logIndex":"0x0","removed":false}]`, }, { f: sys.NewRangeFilter(1, 10, nil, [][]common.Hash{{hash1, hash2}}), - want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x2","transactionHash":"0xa8028c655b6423204c8edfbc339f57b042d6bec2b6a61145d76b7c08b4cccd42","transactionIndex":"0x0","blockHash":"0x24417bb49ce44cfad65da68f33b510bf2a129c0d89ccf06acb6958b8585ccf34","logIndex":"0x0","removed":false},{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x2","transactionHash":"0xdba3e2ea9a7d690b722d70ee605fd67ba4c00d1d3aecd5cf187a7b92ad8eb3df","transactionIndex":"0x1","blockHash":"0x24417bb49ce44cfad65da68f33b510bf2a129c0d89ccf06acb6958b8585ccf34","logIndex":"0x1","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0xdefe471992a07a02acdfbe33edaae22fbb86d7d3cec3f1b8e4e77702fb3acc1d","transactionIndex":"0x0","blockHash":"0x7a7556792ca7d37882882e2b001fe14833eaf81c2c7f865c9c771ec37a024f6b","logIndex":"0x0","removed":false}]`, + want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x2","transactionHash":"0xa8028c655b6423204c8edfbc339f57b042d6bec2b6a61145d76b7c08b4cccd42","transactionIndex":"0x0","blockHash":"0x050509686626eedc759e82b52ee87a22d760e1242b2ab81d6b95e24f4d12cd97","logIndex":"0x0","removed":false},{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x2","transactionHash":"0xdba3e2ea9a7d690b722d70ee605fd67ba4c00d1d3aecd5cf187a7b92ad8eb3df","transactionIndex":"0x1","blockHash":"0x050509686626eedc759e82b52ee87a22d760e1242b2ab81d6b95e24f4d12cd97","logIndex":"0x1","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0x8347f069b3cefebee185fa93df3086c284f8a2b2088d9b7030c970e8d48758b5","transactionIndex":"0x0","blockHash":"0xe5e2e1696398a43b3a464afcfee2f46271ee90800527a4abe50155a7666e5e7c","logIndex":"0x0","removed":false}]`, }, { f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), nil, [][]common.Hash{{common.BytesToHash([]byte("fail"))}}), }, { @@ -304,15 +304,7 @@ func TestFilters(t *testing.T) { f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), nil, [][]common.Hash{{common.BytesToHash([]byte("fail"))}, {hash1}}), }, { f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.LatestBlockNumber), nil, nil), - want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0xb360bad5265261c075ece02d3bf0e39498a6a76310482cdfd90588748e6c5ee0","logIndex":"0x0","removed":false}]`, - }, { - f: sys.NewRangeFilter(int64(rpc.FinalizedBlockNumber), int64(rpc.LatestBlockNumber), nil, nil), - want: `[{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696333"],"data":"0x","blockNumber":"0x3e7","transactionHash":"0x53e3675800c6908424b61b35a44e51ca4c73ca603e58a65b32c67968b4f42200","transactionIndex":"0x0","blockHash":"0x2e4620a2b426b0612ec6cad9603f466723edaed87f98c9137405dd4f7a2409ff","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0xb360bad5265261c075ece02d3bf0e39498a6a76310482cdfd90588748e6c5ee0","logIndex":"0x0","removed":false}]`, - }, { - f: sys.NewRangeFilter(int64(rpc.FinalizedBlockNumber), int64(rpc.FinalizedBlockNumber), nil, nil), - want: `[{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696333"],"data":"0x","blockNumber":"0x3e7","transactionHash":"0x53e3675800c6908424b61b35a44e51ca4c73ca603e58a65b32c67968b4f42200","transactionIndex":"0x0","blockHash":"0x2e4620a2b426b0612ec6cad9603f466723edaed87f98c9137405dd4f7a2409ff","logIndex":"0x0","removed":false}]`, - }, { - f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.FinalizedBlockNumber), nil, nil), + want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0x4d2c709f2f80120b55f5fa8d9a55d15abadbefceaf00102ed942db8ebfbc0cfb","logIndex":"0x0","removed":false}]`, }, { f: sys.NewRangeFilter(int64(rpc.SafeBlockNumber), int64(rpc.LatestBlockNumber), nil, nil), err: "safe header not found", @@ -324,10 +316,10 @@ func TestFilters(t *testing.T) { err: "safe header not found", }, { f: sys.NewRangeFilter(int64(rpc.PendingBlockNumber), int64(rpc.PendingBlockNumber), nil, nil), - want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696335"],"data":"0x","blockNumber":"0x3e9","transactionHash":"0x4110587c1b8d86edc85dce929a34127f1cb8809515a9f177c91c866de3eb0638","transactionIndex":"0x0","blockHash":"0xc7245899e5817f16fa99cf5ad2d9c1e4b98443a565a673ec9c764640443ef037","logIndex":"0x0","removed":false}]`, + want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696335"],"data":"0x","blockNumber":"0x3e9","transactionHash":"0x4110587c1b8d86edc85dce929a34127f1cb8809515a9f177c91c866de3eb0638","transactionIndex":"0x0","blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","logIndex":"0x0","removed":false}]`, }, { f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.PendingBlockNumber), nil, nil), - want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0xb360bad5265261c075ece02d3bf0e39498a6a76310482cdfd90588748e6c5ee0","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696335"],"data":"0x","blockNumber":"0x3e9","transactionHash":"0x4110587c1b8d86edc85dce929a34127f1cb8809515a9f177c91c866de3eb0638","transactionIndex":"0x0","blockHash":"0xc7245899e5817f16fa99cf5ad2d9c1e4b98443a565a673ec9c764640443ef037","logIndex":"0x0","removed":false}]`, + want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0x4d2c709f2f80120b55f5fa8d9a55d15abadbefceaf00102ed942db8ebfbc0cfb","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696335"],"data":"0x","blockNumber":"0x3e9","transactionHash":"0x4110587c1b8d86edc85dce929a34127f1cb8809515a9f177c91c866de3eb0638","transactionIndex":"0x0","blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","logIndex":"0x0","removed":false}]`, }, { f: sys.NewRangeFilter(int64(rpc.PendingBlockNumber), int64(rpc.LatestBlockNumber), nil, nil), err: "invalid block range", From d213640d36f5a9094928fde3a58dcecf1a350b5a Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 12 Jul 2023 21:39:26 +0700 Subject: [PATCH 77/82] Fix downloader unit tests --- core/genesis.go | 3 +- eth/downloader/downloader_test.go | 644 ++++++++++++++++-------------- 2 files changed, 351 insertions(+), 296 deletions(-) diff --git a/core/genesis.go b/core/genesis.go index bd336398aa..6c236c75ae 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -315,7 +315,8 @@ func (g *Genesis) MustCommit(db ethdb.Database) *types.Block { func GenesisBlockForTesting(db ethdb.Database, addr common.Address, balance *big.Int) *types.Block { g := Genesis{ Alloc: GenesisAlloc{addr: {Balance: balance}}, - BaseFee: new(big.Int).SetUint64(params.InitialBaseFee), + BaseFee: new(big.Int).SetUint64(0), + Config: params.TestChainConfig, } return g.MustCommit(db) } diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 4145926173..2dc3a2695b 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -25,13 +25,11 @@ import ( "testing" "time" - "github.com/tomochain/tomochain" "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/consensus/ethash" "github.com/tomochain/tomochain/core" "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/types" - "github.com/tomochain/tomochain/core/vm" "github.com/tomochain/tomochain/ethdb" "github.com/tomochain/tomochain/event" "github.com/tomochain/tomochain/params" @@ -67,37 +65,34 @@ type downloadTester struct { peerMissingStates map[string]map[common.Hash]bool // State entries that fast sync should not return - chain *core.BlockChain - peers map[string]*downloadTesterPeer - lock sync.RWMutex + lock sync.RWMutex } // newTester creates a new downloader test mocker. -func newTester(t *testing.T) *downloadTester { - return newTesterWithNotification(t, nil) -} +func newTester() *downloadTester { + testdb := rawdb.NewMemoryDatabase() + genesis := core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000)) -// newTester creates a new downloader test mocker. -func newTesterWithNotification(t *testing.T, success func()) *downloadTester { - db := rawdb.NewMemoryDatabase() - t.Cleanup(func() { - db.Close() - }) - gspec := &core.Genesis{ - Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, - BaseFee: big.NewInt(params.InitialBaseFee), - } - gspec.MustCommit(db) - chain, err := core.NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) - if err != nil { - panic(err) - } tester := &downloadTester{ - chain: chain, - peers: make(map[string]*downloadTesterPeer), - } - tester.downloader = New(FullSync, db, new(event.TypeMux), tester.chain, nil, tester.dropPeer) + genesis: genesis, + peerDb: testdb, + ownHashes: []common.Hash{genesis.Hash()}, + ownHeaders: map[common.Hash]*types.Header{genesis.Hash(): genesis.Header()}, + ownBlocks: map[common.Hash]*types.Block{genesis.Hash(): genesis}, + ownReceipts: map[common.Hash]types.Receipts{genesis.Hash(): nil}, + ownChainTd: map[common.Hash]*big.Int{genesis.Hash(): genesis.Difficulty()}, + peerHashes: make(map[string][]common.Hash), + peerHeaders: make(map[string]map[common.Hash]*types.Header), + peerBlocks: make(map[string]map[common.Hash]*types.Block), + peerReceipts: make(map[string]map[common.Hash]types.Receipts), + peerChainTds: make(map[string]map[common.Hash]*big.Int), + peerMissingStates: make(map[string]map[common.Hash]bool), + } + tester.stateDb = rawdb.NewMemoryDatabase() + tester.stateDb.Put(genesis.Root().Bytes(), []byte{0x00}) + + tester.downloader = New(FullSync, tester.stateDb, new(event.TypeMux), tester, nil, tester.dropPeer) + return tester } @@ -387,22 +382,8 @@ func (dl *downloadTester) Rollback(hashes []common.Hash) { } // newPeer registers a new block download source into the downloader. -func (dl *downloadTester) newPeer(id string, version uint, blocks []*types.Block) *downloadTesterPeer { - dl.lock.Lock() - defer dl.lock.Unlock() - - peer := &downloadTesterPeer{ - dl: dl, - id: id, - chain: newTestBlockchain(blocks), - withholdHeaders: make(map[common.Hash]struct{}), - } - dl.peers[id] = peer - - if err := dl.downloader.RegisterPeer(id, int(version), peer); err != nil { - panic(err) - } - return peer +func (dl *downloadTester) newPeer(id string, version int, hashes []common.Hash, headers map[common.Hash]*types.Header, blocks map[common.Hash]*types.Block, receipts map[common.Hash]types.Receipts) error { + return dl.newSlowPeer(id, version, hashes, headers, blocks, receipts, 0) } // newSlowPeer registers a new block download source into the downloader, with a @@ -474,12 +455,10 @@ func (dl *downloadTester) dropPeer(id string) { func (dl *downloadTester) Config() *params.ChainConfig { return params.TestChainConfig } type downloadTesterPeer struct { - dl *downloadTester - id string - chain *core.BlockChain - delay time.Duration - lock sync.RWMutex - withholdHeaders map[common.Hash]struct{} + dl *downloadTester + id string + delay time.Duration + lock sync.RWMutex } // setDelay is a thread safe setter for the network delay value. @@ -687,21 +666,23 @@ func TestCanonicalSynchronisation64Light(t *testing.T) { testCanonicalSynchronisation(t, 64, LightSync) } -func testCanonicalSynchronisation(t *testing.T, protocol uint, mode SyncMode) { +func testCanonicalSynchronisation(t *testing.T, protocol int, mode SyncMode) { t.Parallel() - tester := newTester(t) + tester := newTester() defer tester.terminate() // Create a small enough block chain to download - chain := testChainBase.shorten(blockCacheMaxItems - 15) - tester.newPeer("peer", protocol, chain.blocks[1:]) + targetBlocks := blockCacheItems - 15 + hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + + tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) // Synchronise with the peer and make sure all relevant data was retrieved if err := tester.sync("peer", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, len(chain.blocks)) + assertOwnChain(t, tester, targetBlocks+1) } // Tests that if a large batch of blocks are being downloaded, it is throttled @@ -712,14 +693,16 @@ func TestThrottling63Fast(t *testing.T) { testThrottling(t, 63, FastSync) } func TestThrottling64Full(t *testing.T) { testThrottling(t, 64, FullSync) } func TestThrottling64Fast(t *testing.T) { testThrottling(t, 64, FastSync) } -func testThrottling(t *testing.T, protocol uint, mode SyncMode) { +func testThrottling(t *testing.T, protocol int, mode SyncMode) { t.Parallel() - tester := newTester(t) + tester := newTester() defer tester.terminate() // Create a long block chain to download and the tester - targetBlocks := len(testChainBase.blocks) - 1 - tester.newPeer("peer", protocol, testChainBase.blocks[1:]) + targetBlocks := 8 * blockCacheItems + hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + + tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) // Wrap the importer to allow stepping blocked, proceed := uint32(0), make(chan struct{}) @@ -797,29 +780,30 @@ func TestForkedSync64Full(t *testing.T) { testForkedSync(t, 64, FullSync) } func TestForkedSync64Fast(t *testing.T) { testForkedSync(t, 64, FastSync) } func TestForkedSync64Light(t *testing.T) { testForkedSync(t, 64, LightSync) } -func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { +func testForkedSync(t *testing.T, protocol int, mode SyncMode) { t.Parallel() - tester := newTester(t) + tester := newTester() defer tester.terminate() // Create a long enough forked chain - chainA := testChainForkLightA.shorten(len(testChainBase.blocks) + 80) - chainB := testChainForkLightB.shorten(len(testChainBase.blocks) + 81) - tester.newPeer("fork A", protocol, chainA.blocks[1:]) - tester.newPeer("fork B", protocol, chainB.blocks[1:]) + common, fork := MaxHashFetch, 2*MaxHashFetch + hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, true) + + tester.newPeer("fork A", protocol, hashesA, headersA, blocksA, receiptsA) + tester.newPeer("fork B", protocol, hashesB, headersB, blocksB, receiptsB) // Synchronise with the peer and make sure all blocks were retrieved if err := tester.sync("fork A", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, len(chainA.blocks)) + assertOwnChain(t, tester, common+fork+1) // Synchronise with the second peer and make sure that fork is pulled too if err := tester.sync("fork B", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, len(chainB.blocks)) + assertOwnForkedChain(t, tester, common+1, []int{common + fork + 1, common + fork + 1}) } // Tests that synchronising against a much shorter but much heavyer fork works @@ -831,26 +815,30 @@ func TestHeavyForkedSync64Full(t *testing.T) { testHeavyForkedSync(t, 64, FullS func TestHeavyForkedSync64Fast(t *testing.T) { testHeavyForkedSync(t, 64, FastSync) } func TestHeavyForkedSync64Light(t *testing.T) { testHeavyForkedSync(t, 64, LightSync) } -func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { - tester := newTester(t) +func testHeavyForkedSync(t *testing.T, protocol int, mode SyncMode) { + t.Parallel() + + tester := newTester() defer tester.terminate() - chainA := testChainForkLightA.shorten(len(testChainBase.blocks) + 80) - chainB := testChainForkHeavy.shorten(len(testChainBase.blocks) + 79) - tester.newPeer("light", protocol, chainA.blocks[1:]) - tester.newPeer("heavy", protocol, chainB.blocks[1:]) + // Create a long enough forked chain + common, fork := MaxHashFetch, 4*MaxHashFetch + hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, false) + + tester.newPeer("light", protocol, hashesA, headersA, blocksA, receiptsA) + tester.newPeer("heavy", protocol, hashesB[fork/2:], headersB, blocksB, receiptsB) // Synchronise with the peer and make sure all blocks were retrieved if err := tester.sync("light", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, len(chainA.blocks)) + assertOwnChain(t, tester, common+fork+1) // Synchronise with the second peer and make sure that fork is pulled too if err := tester.sync("heavy", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, len(chainB.blocks)) + assertOwnForkedChain(t, tester, common+1, []int{common + fork + 1, common + fork/2 + 1}) } // Tests that chain forks are contained within a certain interval of the current @@ -863,20 +851,24 @@ func TestBoundedForkedSync64Full(t *testing.T) { testBoundedForkedSync(t, 64, F func TestBoundedForkedSync64Fast(t *testing.T) { testBoundedForkedSync(t, 64, FastSync) } func TestBoundedForkedSync64Light(t *testing.T) { testBoundedForkedSync(t, 64, LightSync) } -func testBoundedForkedSync(t *testing.T, protocol uint, mode SyncMode) { - tester := newTester(t) +func testBoundedForkedSync(t *testing.T, protocol int, mode SyncMode) { + t.Parallel() + + tester := newTester() defer tester.terminate() - chainA := testChainForkLightA - chainB := testChainForkLightB - tester.newPeer("original", protocol, chainA.blocks[1:]) - tester.newPeer("rewriter", protocol, chainB.blocks[1:]) + // Create a long enough forked chain + common, fork := 13, int(MaxForkAncestry+17) + hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, true) + + tester.newPeer("original", protocol, hashesA, headersA, blocksA, receiptsA) + tester.newPeer("rewriter", protocol, hashesB, headersB, blocksB, receiptsB) // Synchronise with the peer and make sure all blocks were retrieved if err := tester.sync("original", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, len(chainA.blocks)) + assertOwnChain(t, tester, common+fork+1) // Synchronise with the second peer and ensure that the fork is rejected to being too old if err := tester.sync("rewriter", nil, mode); err != errInvalidAncestor { @@ -894,22 +886,25 @@ func TestBoundedHeavyForkedSync64Full(t *testing.T) { testBoundedHeavyForkedSyn func TestBoundedHeavyForkedSync64Fast(t *testing.T) { testBoundedHeavyForkedSync(t, 64, FastSync) } func TestBoundedHeavyForkedSync64Light(t *testing.T) { testBoundedHeavyForkedSync(t, 64, LightSync) } -func testBoundedHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { - tester := newTester(t) +func testBoundedHeavyForkedSync(t *testing.T, protocol int, mode SyncMode) { + t.Parallel() + + tester := newTester() defer tester.terminate() // Create a long enough forked chain - chainA := testChainForkLightA - chainB := testChainForkHeavy - tester.newPeer("original", protocol, chainA.blocks[1:]) + common, fork := 13, int(MaxForkAncestry+17) + hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, false) + + tester.newPeer("original", protocol, hashesA, headersA, blocksA, receiptsA) + tester.newPeer("heavy-rewriter", protocol, hashesB[MaxForkAncestry-17:], headersB, blocksB, receiptsB) // Root the fork below the ancestor limit // Synchronise with the peer and make sure all blocks were retrieved if err := tester.sync("original", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, len(chainA.blocks)) + assertOwnChain(t, tester, common+fork+1) - tester.newPeer("heavy-rewriter", protocol, chainB.blocks[1:]) // Synchronise with the second peer and ensure that the fork is rejected to being too old if err := tester.sync("heavy-rewriter", nil, mode); err != errInvalidAncestor { t.Fatalf("sync failure mismatch: have %v, want %v", err, errInvalidAncestor) @@ -921,7 +916,7 @@ func testBoundedHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { func TestInactiveDownloader62(t *testing.T) { t.Parallel() - tester := newTester(t) + tester := newTester() defer tester.terminate() // Check that neither block headers nor bodies are accepted @@ -938,7 +933,7 @@ func TestInactiveDownloader62(t *testing.T) { func TestInactiveDownloader63(t *testing.T) { t.Parallel() - tester := newTester(t) + tester := newTester() defer tester.terminate() // Check that neither block headers nor bodies are accepted @@ -961,12 +956,23 @@ func TestCancel64Full(t *testing.T) { testCancel(t, 64, FullSync) } func TestCancel64Fast(t *testing.T) { testCancel(t, 64, FastSync) } func TestCancel64Light(t *testing.T) { testCancel(t, 64, LightSync) } -func testCancel(t *testing.T, protocol uint, mode SyncMode) { - tester := newTester(t) +func testCancel(t *testing.T, protocol int, mode SyncMode) { + t.Parallel() + + tester := newTester() defer tester.terminate() - chain := testChainBase.shorten(MaxHeaderFetch) - tester.newPeer("peer", protocol, chain.blocks[1:]) + // Create a small enough block chain to download and the tester + targetBlocks := blockCacheItems - 15 + if targetBlocks >= MaxHashFetch { + targetBlocks = MaxHashFetch - 15 + } + if targetBlocks >= MaxHeaderFetch { + targetBlocks = MaxHeaderFetch - 15 + } + hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + + tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) // Make sure canceling works with a pristine downloader tester.downloader.Cancel() @@ -991,22 +997,25 @@ func TestMultiSynchronisation64Full(t *testing.T) { testMultiSynchronisation(t, func TestMultiSynchronisation64Fast(t *testing.T) { testMultiSynchronisation(t, 64, FastSync) } func TestMultiSynchronisation64Light(t *testing.T) { testMultiSynchronisation(t, 64, LightSync) } -func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { - tester := newTester(t) +func testMultiSynchronisation(t *testing.T, protocol int, mode SyncMode) { + t.Parallel() + + tester := newTester() defer tester.terminate() // Create various peers with various parts of the chain targetPeers := 8 - chain := testChainBase.shorten(targetPeers * 100) + targetBlocks := targetPeers*blockCacheItems - 15 + hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) for i := 0; i < targetPeers; i++ { id := fmt.Sprintf("peer #%d", i) - tester.newPeer(id, protocol, chain.shorten(len(chain.blocks) / (i + 1)).blocks[1:]) + tester.newPeer(id, protocol, hashes[i*blockCacheItems:], headers, blocks, receipts) } if err := tester.sync("peer #0", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, len(chain.blocks)) + assertOwnChain(t, tester, targetBlocks+1) } // Tests that synchronisations behave well in multi-version protocol environments @@ -1018,28 +1027,31 @@ func TestMultiProtoSynchronisation64Full(t *testing.T) { testMultiProtoSync(t, func TestMultiProtoSynchronisation64Fast(t *testing.T) { testMultiProtoSync(t, 64, FastSync) } func TestMultiProtoSynchronisation64Light(t *testing.T) { testMultiProtoSync(t, 64, LightSync) } -func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { - tester := newTester(t) +func testMultiProtoSync(t *testing.T, protocol int, mode SyncMode) { + t.Parallel() + + tester := newTester() defer tester.terminate() // Create a small enough block chain to download - chain := testChainBase.shorten(blockCacheMaxItems - 15) + targetBlocks := blockCacheItems - 15 + hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) // Create peers of every type - tester.newPeer("peer 62", 62, chain.blocks[1:]) - tester.newPeer("peer 63", 63, chain.blocks[1:]) - tester.newPeer("peer 64", 64, chain.blocks[1:]) + tester.newPeer("peer 62", 62, hashes, headers, blocks, nil) + tester.newPeer("peer 63", 63, hashes, headers, blocks, receipts) + tester.newPeer("peer 64", 64, hashes, headers, blocks, receipts) // Synchronise with the requested peer and make sure all blocks were retrieved if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, len(chain.blocks)) + assertOwnChain(t, tester, targetBlocks+1) // Check that no peers have been dropped off - for _, version := range []int{66, 67} { + for _, version := range []int{62, 63, 64} { peer := fmt.Sprintf("peer %d", version) - if _, ok := tester.peers[peer]; !ok { + if _, ok := tester.peerHashes[peer]; !ok { t.Errorf("%s dropped", peer) } } @@ -1054,13 +1066,17 @@ func TestEmptyShortCircuit64Full(t *testing.T) { testEmptyShortCircuit(t, 64, F func TestEmptyShortCircuit64Fast(t *testing.T) { testEmptyShortCircuit(t, 64, FastSync) } func TestEmptyShortCircuit64Light(t *testing.T) { testEmptyShortCircuit(t, 64, LightSync) } -func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { - tester := newTester(t) +func testEmptyShortCircuit(t *testing.T, protocol int, mode SyncMode) { + t.Parallel() + + tester := newTester() defer tester.terminate() // Create a block chain to download - chain := testChainBase - tester.newPeer("peer", protocol, chain.blocks[1:]) + targetBlocks := 2*blockCacheItems - 15 + hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + + tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) // Instrument the downloader to signal body requests bodiesHave, receiptsHave := int32(0), int32(0) @@ -1074,17 +1090,17 @@ func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { if err := tester.sync("peer", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, len(chain.blocks)) + assertOwnChain(t, tester, targetBlocks+1) // Validate the number of block bodies that should have been requested bodiesNeeded, receiptsNeeded := 0, 0 - for _, block := range chain.blocks[1:] { - if mode != LightSync && (len(block.Transactions()) > 0 || len(block.Uncles()) > 0) { + for _, block := range blocks { + if mode != LightSync && block != tester.genesis && (len(block.Transactions()) > 0 || len(block.Uncles()) > 0) { bodiesNeeded++ } } - for _, block := range chain.blocks[1:] { - if mode == FastSync && len(block.Transactions()) > 0 { + for _, receipt := range receipts { + if mode == FastSync && len(receipt) > 0 { receiptsNeeded++ } } @@ -1105,24 +1121,30 @@ func TestMissingHeaderAttack64Full(t *testing.T) { testMissingHeaderAttack(t, 6 func TestMissingHeaderAttack64Fast(t *testing.T) { testMissingHeaderAttack(t, 64, FastSync) } func TestMissingHeaderAttack64Light(t *testing.T) { testMissingHeaderAttack(t, 64, LightSync) } -func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { - tester := newTester(t) +func testMissingHeaderAttack(t *testing.T, protocol int, mode SyncMode) { + t.Parallel() + + tester := newTester() defer tester.terminate() - chain := testChainBase.shorten(blockCacheMaxItems - 15) + // Create a small enough block chain to download + targetBlocks := blockCacheItems - 15 + hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - attacker := tester.newPeer("attack", protocol, chain.blocks[1:]) - attacker.withholdHeaders[chain.blocks[len(chain.blocks)/2-1].Hash()] = struct{}{} + // Attempt a full sync with an attacker feeding gapped headers + tester.newPeer("attack", protocol, hashes, headers, blocks, receipts) + missing := targetBlocks / 2 + delete(tester.peerHeaders["attack"], hashes[missing]) if err := tester.sync("attack", nil, mode); err == nil { t.Fatalf("succeeded attacker synchronisation") } // Synchronise with the valid peer and make sure sync succeeds - tester.newPeer("valid", protocol, chain.blocks[1:]) + tester.newPeer("valid", protocol, hashes, headers, blocks, receipts) if err := tester.sync("valid", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, len(chain.blocks)) + assertOwnChain(t, tester, targetBlocks+1) } // Tests that if requested headers are shifted (i.e. first is missing), the queue @@ -1134,25 +1156,31 @@ func TestShiftedHeaderAttack64Full(t *testing.T) { testShiftedHeaderAttack(t, 6 func TestShiftedHeaderAttack64Fast(t *testing.T) { testShiftedHeaderAttack(t, 64, FastSync) } func TestShiftedHeaderAttack64Light(t *testing.T) { testShiftedHeaderAttack(t, 64, LightSync) } -func testShiftedHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { - tester := newTester(t) +func testShiftedHeaderAttack(t *testing.T, protocol int, mode SyncMode) { + t.Parallel() + + tester := newTester() defer tester.terminate() - chain := testChainBase.shorten(blockCacheMaxItems - 15) + // Create a small enough block chain to download + targetBlocks := blockCacheItems - 15 + hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) // Attempt a full sync with an attacker feeding shifted headers - attacker := tester.newPeer("attack", protocol, chain.blocks[1:]) - attacker.withholdHeaders[chain.blocks[1].Hash()] = struct{}{} + tester.newPeer("attack", protocol, hashes, headers, blocks, receipts) + delete(tester.peerHeaders["attack"], hashes[len(hashes)-2]) + delete(tester.peerBlocks["attack"], hashes[len(hashes)-2]) + delete(tester.peerReceipts["attack"], hashes[len(hashes)-2]) if err := tester.sync("attack", nil, mode); err == nil { t.Fatalf("succeeded attacker synchronisation") } // Synchronise with the valid peer and make sure sync succeeds - tester.newPeer("valid", protocol, chain.blocks[1:]) + tester.newPeer("valid", protocol, hashes, headers, blocks, receipts) if err := tester.sync("valid", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, len(chain.blocks)) + assertOwnChain(t, tester, targetBlocks+1) } // Tests that upon detecting an invalid header, the recent ones are rolled back @@ -1162,78 +1190,87 @@ func TestInvalidHeaderRollback63Fast(t *testing.T) { testInvalidHeaderRollback( func TestInvalidHeaderRollback64Fast(t *testing.T) { testInvalidHeaderRollback(t, 64, FastSync) } func TestInvalidHeaderRollback64Light(t *testing.T) { testInvalidHeaderRollback(t, 64, LightSync) } -func testInvalidHeaderRollback(t *testing.T, protocol uint, mode SyncMode) { - tester := newTester(t) +func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { + t.Parallel() + + tester := newTester() defer tester.terminate() // Create a small enough block chain to download targetBlocks := 3*fsHeaderSafetyNet + 256 + fsMinFullBlocks - chain := testChainBase.shorten(targetBlocks) + hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) // Attempt to sync with an attacker that feeds junk during the fast sync phase. // This should result in the last fsHeaderSafetyNet headers being rolled back. + tester.newPeer("fast-attack", protocol, hashes, headers, blocks, receipts) missing := fsHeaderSafetyNet + MaxHeaderFetch + 1 - - fastAttacker := tester.newPeer("fast-attack", protocol, chain.blocks[1:]) - fastAttacker.withholdHeaders[chain.blocks[missing].Hash()] = struct{}{} + delete(tester.peerHeaders["fast-attack"], hashes[len(hashes)-missing]) if err := tester.sync("fast-attack", nil, mode); err == nil { t.Fatalf("succeeded fast attacker synchronisation") } - if head := tester.chain.CurrentHeader().Number.Int64(); int(head) > MaxHeaderFetch { + if head := tester.CurrentHeader().Number.Int64(); int(head) > MaxHeaderFetch { t.Errorf("rollback head mismatch: have %v, want at most %v", head, MaxHeaderFetch) } // Attempt to sync with an attacker that feeds junk during the block import phase. // This should result in both the last fsHeaderSafetyNet number of headers being // rolled back, and also the pivot point being reverted to a non-block status. + tester.newPeer("block-attack", protocol, hashes, headers, blocks, receipts) missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1 - - blockAttacker := tester.newPeer("block-attack", protocol, chain.blocks[1:]) - fastAttacker.withholdHeaders[chain.blocks[missing].Hash()] = struct{}{} // Make sure the fast-attacker doesn't fill in - blockAttacker.withholdHeaders[chain.blocks[missing].Hash()] = struct{}{} + delete(tester.peerHeaders["fast-attack"], hashes[len(hashes)-missing]) // Make sure the fast-attacker doesn't fill in + delete(tester.peerHeaders["block-attack"], hashes[len(hashes)-missing]) if err := tester.sync("block-attack", nil, mode); err == nil { t.Fatalf("succeeded block attacker synchronisation") } - if head := tester.chain.CurrentHeader().Number.Int64(); int(head) > 2*fsHeaderSafetyNet+MaxHeaderFetch { + if head := tester.CurrentHeader().Number.Int64(); int(head) > 2*fsHeaderSafetyNet+MaxHeaderFetch { t.Errorf("rollback head mismatch: have %v, want at most %v", head, 2*fsHeaderSafetyNet+MaxHeaderFetch) } if mode == FastSync { - if head := tester.chain.CurrentBlock().Number().Uint64(); head != 0 { + if head := tester.CurrentBlock().NumberU64(); head != 0 { t.Errorf("fast sync pivot block #%d not rolled back", head) } } // Attempt to sync with an attacker that withholds promised blocks after the // fast sync pivot point. This could be a trial to leave the node with a bad // but already imported pivot block. - withholdAttacker := tester.newPeer("withhold-attack", protocol, chain.blocks[1:]) + tester.newPeer("withhold-attack", protocol, hashes, headers, blocks, receipts) + missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1 tester.downloader.syncInitHook = func(uint64, uint64) { - for i := missing; i < len(chain.blocks); i++ { - withholdAttacker.withholdHeaders[chain.blocks[i].Hash()] = struct{}{} + for i := missing; i <= len(hashes); i++ { + delete(tester.peerHeaders["withhold-attack"], hashes[len(hashes)-i]) } tester.downloader.syncInitHook = nil } + if err := tester.sync("withhold-attack", nil, mode); err == nil { t.Fatalf("succeeded withholding attacker synchronisation") } - if head := tester.chain.CurrentHeader().Number.Int64(); int(head) > 2*fsHeaderSafetyNet+MaxHeaderFetch { + if head := tester.CurrentHeader().Number.Int64(); int(head) > 2*fsHeaderSafetyNet+MaxHeaderFetch { t.Errorf("rollback head mismatch: have %v, want at most %v", head, 2*fsHeaderSafetyNet+MaxHeaderFetch) } if mode == FastSync { - if head := tester.chain.CurrentBlock().Number().Uint64(); head != 0 { + if head := tester.CurrentBlock().NumberU64(); head != 0 { t.Errorf("fast sync pivot block #%d not rolled back", head) } } - // Synchronise with the valid peer and make sure sync succeeds. Since the last rollback - // should also disable fast syncing for this process, verify that we did a fresh full - // sync. Note, we can't assert anything about the receipts since we won't purge the - // database of them, hence we can't use assertOwnChain. - tester.newPeer("valid", protocol, chain.blocks[1:]) + // Synchronise with the valid peer and make sure sync succeeds. Since the last + // rollback should also disable fast syncing for this process, verify that we + // did a fresh full sync. Note, we can't assert anything about the receipts + // since we won't purge the database of them, hence we can't use assertOwnChain. + tester.newPeer("valid", protocol, hashes, headers, blocks, receipts) if err := tester.sync("valid", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, len(chain.blocks)) + if hs := len(tester.ownHeaders); hs != len(headers) { + t.Fatalf("synchronised headers mismatch: have %v, want %v", hs, len(headers)) + } + if mode != LightSync { + if bs := len(tester.ownBlocks); bs != len(blocks) { + t.Fatalf("synchronised blocks mismatch: have %v, want %v", bs, len(blocks)) + } + } } // Tests that a peer advertising an high TD doesn't get to stall the downloader @@ -1245,12 +1282,15 @@ func TestHighTDStarvationAttack64Full(t *testing.T) { testHighTDStarvationAttac func TestHighTDStarvationAttack64Fast(t *testing.T) { testHighTDStarvationAttack(t, 64, FastSync) } func TestHighTDStarvationAttack64Light(t *testing.T) { testHighTDStarvationAttack(t, 64, LightSync) } -func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { - tester := newTester(t) +func testHighTDStarvationAttack(t *testing.T, protocol int, mode SyncMode) { + t.Parallel() + + tester := newTester() defer tester.terminate() - chain := testChainBase.shorten(1) - tester.newPeer("attack", protocol, chain.blocks[1:]) + hashes, headers, blocks, receipts := tester.makeChain(0, 0, tester.genesis, nil, false) + tester.newPeer("attack", protocol, []common.Hash{hashes[0]}, headers, blocks, receipts) + if err := tester.sync("attack", big.NewInt(1000000), mode); err != errStallingPeer { t.Fatalf("synchronisation error mismatch: have %v, want %v", err, errStallingPeer) } @@ -1261,7 +1301,7 @@ func TestBlockHeaderAttackerDropping62(t *testing.T) { testBlockHeaderAttackerDr func TestBlockHeaderAttackerDropping63(t *testing.T) { testBlockHeaderAttackerDropping(t, 63) } func TestBlockHeaderAttackerDropping64(t *testing.T) { testBlockHeaderAttackerDropping(t, 64) } -func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { +func testBlockHeaderAttackerDropping(t *testing.T, protocol int) { t.Parallel() // Define the disconnection requirement for individual hash fetch errors @@ -1291,15 +1331,16 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { {errCancelContentProcessing, false}, // Synchronisation was canceled, origin may be innocent, don't drop } // Run the tests and check disconnection status - tester := newTester(t) + tester := newTester() defer tester.terminate() - chain := testChainBase.shorten(1) for i, tt := range tests { - // Register a new peer and ensure its presence + // Register a new peer and ensure it's presence id := fmt.Sprintf("test %d", i) - tester.newPeer(id, protocol, chain.blocks[1:]) - if _, ok := tester.peers[id]; !ok { + if err := tester.newPeer(id, protocol, []common.Hash{tester.genesis.Hash()}, nil, nil, nil); err != nil { + t.Fatalf("test %d: failed to register new peer: %v", i, err) + } + if _, ok := tester.peerHashes[id]; !ok { t.Fatalf("test %d: registered peer not found", i) } // Simulate a synchronisation and check the required result @@ -1321,11 +1362,15 @@ func TestSyncProgress64Full(t *testing.T) { testSyncProgress(t, 64, FullSync) } func TestSyncProgress64Fast(t *testing.T) { testSyncProgress(t, 64, FastSync) } func TestSyncProgress64Light(t *testing.T) { testSyncProgress(t, 64, LightSync) } -func testSyncProgress(t *testing.T, protocol uint, mode SyncMode) { - tester := newTester(t) +func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { + t.Parallel() + + tester := newTester() defer tester.terminate() - chain := testChainBase.shorten(blockCacheMaxItems - 15) + // Create a small enough block chain to download + targetBlocks := blockCacheItems - 15 + hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) // Set a sync init hook to catch progress changes starting := make(chan struct{}) @@ -1335,10 +1380,12 @@ func testSyncProgress(t *testing.T, protocol uint, mode SyncMode) { starting <- struct{}{} <-progress } - checkProgress(t, tester.downloader, "pristine", tomochain.SyncProgress{}) - + // Retrieve the sync progress and ensure they are zero (pristine sync) + if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { + t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) + } // Synchronise half the blocks and check initial progress - tester.newPeer("peer-half", protocol, chain.shorten(len(chain.blocks) / 2).blocks[1:]) + tester.newPeer("peer-half", protocol, hashes[targetBlocks/2:], headers, blocks, receipts) pending := new(sync.WaitGroup) pending.Add(1) @@ -1349,15 +1396,16 @@ func testSyncProgress(t *testing.T, protocol uint, mode SyncMode) { } }() <-starting - checkProgress(t, tester.downloader, "initial", tomochain.SyncProgress{ - HighestBlock: uint64(len(chain.blocks)/2 - 1), - }) + if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks/2+1) { + t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks/2+1) + } progress <- struct{}{} pending.Wait() // Synchronise all the blocks and check continuation progress - tester.newPeer("peer-full", protocol, chain.blocks[1:]) + tester.newPeer("peer-full", protocol, hashes, headers, blocks, receipts) pending.Add(1) + go func() { defer pending.Done() if err := tester.sync("peer-full", nil, mode); err != nil { @@ -1365,29 +1413,15 @@ func testSyncProgress(t *testing.T, protocol uint, mode SyncMode) { } }() <-starting - checkProgress(t, tester.downloader, "completing", tomochain.SyncProgress{ - StartingBlock: uint64(len(chain.blocks)/2 - 1), - CurrentBlock: uint64(len(chain.blocks)/2 - 1), - HighestBlock: uint64(len(chain.blocks) - 1), - }) - - // Check final progress after successful sync + if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(targetBlocks/2+1) || progress.CurrentBlock != uint64(targetBlocks/2+1) || progress.HighestBlock != uint64(targetBlocks) { + t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2+1, targetBlocks/2+1, targetBlocks) + } progress <- struct{}{} pending.Wait() - checkProgress(t, tester.downloader, "final", tomochain.SyncProgress{ - StartingBlock: uint64(len(chain.blocks)/2 - 1), - CurrentBlock: uint64(len(chain.blocks) - 1), - HighestBlock: uint64(len(chain.blocks) - 1), - }) -} - -func checkProgress(t *testing.T, d *Downloader, stage string, want tomochain.SyncProgress) { - // Mark this method as a helper to report errors at callsite, not in here - t.Helper() - p := d.Progress() - if p.StartingBlock != want.StartingBlock || p.CurrentBlock != want.CurrentBlock || p.HighestBlock != want.HighestBlock { - t.Fatalf("%s progress mismatch:\nhave %+v\nwant %+v", stage, p, want) + // Check final progress after successful sync + if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(targetBlocks/2+1) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { + t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2+1, targetBlocks, targetBlocks) } } @@ -1401,12 +1435,15 @@ func TestForkedSyncProgress64Full(t *testing.T) { testForkedSyncProgress(t, 64, func TestForkedSyncProgress64Fast(t *testing.T) { testForkedSyncProgress(t, 64, FastSync) } func TestForkedSyncProgress64Light(t *testing.T) { testForkedSyncProgress(t, 64, LightSync) } -func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { - tester := newTester(t) +func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { + t.Parallel() + + tester := newTester() defer tester.terminate() - chainA := testChainForkLightA.shorten(len(testChainBase.blocks) + MaxHeaderFetch) - chainB := testChainForkLightB.shorten(len(testChainBase.blocks) + MaxHeaderFetch) + // Create a forked chain to simulate origin revertal + common, fork := MaxHashFetch, 2*MaxHashFetch + hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, true) // Set a sync init hook to catch progress changes starting := make(chan struct{}) @@ -1416,12 +1453,15 @@ func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { starting <- struct{}{} <-progress } - checkProgress(t, tester.downloader, "pristine", tomochain.SyncProgress{}) - + // Retrieve the sync progress and ensure they are zero (pristine sync) + if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { + t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) + } // Synchronise with one of the forks and check progress - tester.newPeer("fork A", protocol, chainA.blocks[1:]) + tester.newPeer("fork A", protocol, hashesA, headersA, blocksA, receiptsA) pending := new(sync.WaitGroup) pending.Add(1) + go func() { defer pending.Done() if err := tester.sync("fork A", nil, mode); err != nil { @@ -1429,10 +1469,9 @@ func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { } }() <-starting - - checkProgress(t, tester.downloader, "initial", tomochain.SyncProgress{ - HighestBlock: uint64(len(chainA.blocks) - 1), - }) + if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(len(hashesA)-1) { + t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, len(hashesA)-1) + } progress <- struct{}{} pending.Wait() @@ -1440,8 +1479,9 @@ func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester.downloader.syncStatsChainOrigin = tester.downloader.syncStatsChainHeight // Synchronise with the second fork and check progress resets - tester.newPeer("fork B", protocol, chainB.blocks[1:]) + tester.newPeer("fork B", protocol, hashesB, headersB, blocksB, receiptsB) pending.Add(1) + go func() { defer pending.Done() if err := tester.sync("fork B", nil, mode); err != nil { @@ -1449,20 +1489,16 @@ func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { } }() <-starting - checkProgress(t, tester.downloader, "forking", tomochain.SyncProgress{ - StartingBlock: uint64(len(testChainBase.blocks)) - 1, - CurrentBlock: uint64(len(chainA.blocks) - 1), - HighestBlock: uint64(len(chainB.blocks) - 1), - }) - - // Check final progress after successful sync + if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(common) || progress.CurrentBlock != uint64(len(hashesA)-1) || progress.HighestBlock != uint64(len(hashesB)-1) { + t.Fatalf("Forking progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, common, len(hashesA)-1, len(hashesB)-1) + } progress <- struct{}{} pending.Wait() - checkProgress(t, tester.downloader, "final", tomochain.SyncProgress{ - StartingBlock: uint64(len(testChainBase.blocks)) - 1, - CurrentBlock: uint64(len(chainB.blocks) - 1), - HighestBlock: uint64(len(chainB.blocks) - 1), - }) + + // Check final progress after successful sync + if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(common) || progress.CurrentBlock != uint64(len(hashesB)-1) || progress.HighestBlock != uint64(len(hashesB)-1) { + t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, common, len(hashesB)-1, len(hashesB)-1) + } } // Tests that if synchronisation is aborted due to some failure, then the progress @@ -1475,11 +1511,15 @@ func TestFailedSyncProgress64Full(t *testing.T) { testFailedSyncProgress(t, 64, func TestFailedSyncProgress64Fast(t *testing.T) { testFailedSyncProgress(t, 64, FastSync) } func TestFailedSyncProgress64Light(t *testing.T) { testFailedSyncProgress(t, 64, LightSync) } -func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { - tester := newTester(t) +func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { + t.Parallel() + + tester := newTester() defer tester.terminate() - chain := testChainBase.shorten(blockCacheMaxItems - 15) + // Create a small enough block chain to download + targetBlocks := blockCacheItems - 15 + hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) // Set a sync init hook to catch progress changes starting := make(chan struct{}) @@ -1489,16 +1529,20 @@ func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { starting <- struct{}{} <-progress } - checkProgress(t, tester.downloader, "pristine", tomochain.SyncProgress{}) - + // Retrieve the sync progress and ensure they are zero (pristine sync) + if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { + t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) + } // Attempt a full sync with a faulty peer - missing := len(chain.blocks)/2 - 1 - - faulter := tester.newPeer("faulty", protocol, chain.blocks[1:]) - faulter.withholdHeaders[chain.blocks[missing].Hash()] = struct{}{} + tester.newPeer("faulty", protocol, hashes, headers, blocks, receipts) + missing := targetBlocks / 2 + delete(tester.peerHeaders["faulty"], hashes[missing]) + delete(tester.peerBlocks["faulty"], hashes[missing]) + delete(tester.peerReceipts["faulty"], hashes[missing]) pending := new(sync.WaitGroup) pending.Add(1) + go func() { defer pending.Done() if err := tester.sync("faulty", nil, mode); err == nil { @@ -1506,17 +1550,16 @@ func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { } }() <-starting - checkProgress(t, tester.downloader, "initial", tomochain.SyncProgress{ - HighestBlock: uint64(len(chain.blocks) - 1), - }) + if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks) { + t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks) + } progress <- struct{}{} pending.Wait() - afterFailedSync := tester.downloader.Progress() - // Synchronise with a good peer and check that the progress origin remind the same - // after a failure - tester.newPeer("valid", protocol, chain.blocks[1:]) + // Synchronise with a good peer and check that the progress origin remind the same after a failure + tester.newPeer("valid", protocol, hashes, headers, blocks, receipts) pending.Add(1) + go func() { defer pending.Done() if err := tester.sync("valid", nil, mode); err != nil { @@ -1524,15 +1567,16 @@ func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { } }() <-starting - checkProgress(t, tester.downloader, "completing", afterFailedSync) - - // Check final progress after successful sync + if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock > uint64(targetBlocks/2) || progress.HighestBlock != uint64(targetBlocks) { + t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/0-%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, targetBlocks/2, targetBlocks) + } progress <- struct{}{} pending.Wait() - checkProgress(t, tester.downloader, "final", tomochain.SyncProgress{ - CurrentBlock: uint64(len(chain.blocks) - 1), - HighestBlock: uint64(len(chain.blocks) - 1), - }) + + // Check final progress after successful sync + if progress := tester.downloader.Progress(); progress.StartingBlock > uint64(targetBlocks/2) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { + t.Fatalf("Final progress mismatch: have %v/%v/%v, want 0-%v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2, targetBlocks, targetBlocks) + } } // Tests that if an attacker fakes a chain height, after the attack is detected, @@ -1544,29 +1588,39 @@ func TestFakedSyncProgress64Full(t *testing.T) { testFakedSyncProgress(t, 64, F func TestFakedSyncProgress64Fast(t *testing.T) { testFakedSyncProgress(t, 64, FastSync) } func TestFakedSyncProgress64Light(t *testing.T) { testFakedSyncProgress(t, 64, LightSync) } -func testFakedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { - tester := newTester(t) +func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) { + t.Parallel() + + tester := newTester() defer tester.terminate() - chain := testChainBase.shorten(blockCacheMaxItems - 15) + // Create a small block chain + targetBlocks := blockCacheItems - 15 + hashes, headers, blocks, receipts := tester.makeChain(targetBlocks+3, 0, tester.genesis, nil, false) // Set a sync init hook to catch progress changes starting := make(chan struct{}) progress := make(chan struct{}) + tester.downloader.syncInitHook = func(origin, latest uint64) { starting <- struct{}{} <-progress } - checkProgress(t, tester.downloader, "pristine", tomochain.SyncProgress{}) - - // Create and sync with an attacker that promises a higher chain than available. - attacker := tester.newPeer("attack", protocol, chain.blocks[1:]) - numMissing := 5 - for i := len(chain.blocks) - 2; i > len(chain.blocks)-numMissing; i-- { - attacker.withholdHeaders[chain.blocks[i].Hash()] = struct{}{} + // Retrieve the sync progress and ensure they are zero (pristine sync) + if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { + t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) + } + // Create and sync with an attacker that promises a higher chain than available + tester.newPeer("attack", protocol, hashes, headers, blocks, receipts) + for i := 1; i < 3; i++ { + delete(tester.peerHeaders["attack"], hashes[i]) + delete(tester.peerBlocks["attack"], hashes[i]) + delete(tester.peerReceipts["attack"], hashes[i]) } + pending := new(sync.WaitGroup) pending.Add(1) + go func() { defer pending.Done() if err := tester.sync("attack", nil, mode); err == nil { @@ -1574,17 +1628,14 @@ func testFakedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { } }() <-starting - checkProgress(t, tester.downloader, "initial", tomochain.SyncProgress{ - HighestBlock: uint64(len(chain.blocks) - 1), - }) + if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks+3) { + t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks+3) + } progress <- struct{}{} pending.Wait() - afterFailedSync := tester.downloader.Progress() - // Synchronise with a good peer and check that the progress height has been reduced to - // the true value. - validChain := chain.shorten(len(chain.blocks) - numMissing) - tester.newPeer("valid", protocol, validChain.blocks[1:]) + // Synchronise with a good peer and check that the progress height has been reduced to the true value + tester.newPeer("valid", protocol, hashes[3:], headers, blocks, receipts) pending.Add(1) go func() { @@ -1594,17 +1645,16 @@ func testFakedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { } }() <-starting - checkProgress(t, tester.downloader, "completing", tomochain.SyncProgress{ - CurrentBlock: afterFailedSync.CurrentBlock, - HighestBlock: uint64(len(validChain.blocks) - 1), - }) - // Check final progress after successful sync. + if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock > uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { + t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/0-%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, targetBlocks, targetBlocks) + } progress <- struct{}{} pending.Wait() - checkProgress(t, tester.downloader, "final", tomochain.SyncProgress{ - CurrentBlock: uint64(len(validChain.blocks) - 1), - HighestBlock: uint64(len(validChain.blocks) - 1), - }) + + // Check final progress after successful sync + if progress := tester.downloader.Progress(); progress.StartingBlock > uint64(targetBlocks) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { + t.Fatalf("Final progress mismatch: have %v/%v/%v, want 0-%v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks, targetBlocks, targetBlocks) + } } // This test reproduces an issue where unexpected deliveries would @@ -1613,7 +1663,7 @@ func testFakedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // and not with the other tests, avoiding intermittent failures. func TestDeliverHeadersHang(t *testing.T) { testCases := []struct { - protocol uint + protocol int syncMode SyncMode }{ {62, FullSync}, @@ -1630,30 +1680,6 @@ func TestDeliverHeadersHang(t *testing.T) { } } -func testDeliverHeadersHang(t *testing.T, protocol uint, mode SyncMode) { - master := newTester(t) - defer master.terminate() - - chain := testChainBase.shorten(15) - - for i := 0; i < 200; i++ { - tester := newTester(t) - tester.peerDb = master.peerDb - tester.newPeer("peer", protocol, chain.blocks[1:]) - - // Whenever the downloader requests headers, flood it with - // a lot of unrequested header deliveries. - tester.downloader.peers.peers["peer"].peer = &floodingTestPeer{ - peer: tester.downloader.peers.peers["peer"].peer, - tester: tester, - } - if err := tester.sync("peer", nil, mode); err != nil { - t.Errorf("test %d: sync failed: %v", i, err) - } - tester.terminate() - } -} - type floodingTestPeer struct { peer Peer tester *downloadTester @@ -1699,3 +1725,31 @@ func (ftp *floodingTestPeer) RequestHeadersByNumber(from uint64, count, skip int } return nil } + +func testDeliverHeadersHang(t *testing.T, protocol int, mode SyncMode) { + t.Parallel() + + master := newTester() + defer master.terminate() + + hashes, headers, blocks, receipts := master.makeChain(5, 0, master.genesis, nil, false) + for i := 0; i < 200; i++ { + tester := newTester() + tester.peerDb = master.peerDb + + tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) + // Whenever the downloader requests headers, flood it with + // a lot of unrequested header deliveries. + tester.downloader.peers.peers["peer"].peer = &floodingTestPeer{ + peer: tester.downloader.peers.peers["peer"].peer, + tester: tester, + } + if err := tester.sync("peer", nil, mode); err != nil { + t.Errorf("test %d: sync failed: %v", i, err) + } + tester.terminate() + + // Flush all goroutines to prevent messing with subsequent tests + tester.downloader.peers.peers["peer"].peer.(*floodingTestPeer).pend.Wait() + } +} From 6ecbedf364f1166efb0df8980bba063e233e622e Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Wed, 12 Jul 2023 22:22:31 +0700 Subject: [PATCH 78/82] Fix eth fetcher unit tests --- eth/fetcher/fetcher_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eth/fetcher/fetcher_test.go b/eth/fetcher/fetcher_test.go index ab7e03aaa1..e6cb26b53b 100644 --- a/eth/fetcher/fetcher_test.go +++ b/eth/fetcher/fetcher_test.go @@ -18,7 +18,6 @@ package fetcher import ( "errors" - "github.com/tomochain/tomochain/core/rawdb" "math/big" "sync" "sync/atomic" @@ -28,6 +27,7 @@ import ( "github.com/tomochain/tomochain/common" "github.com/tomochain/tomochain/consensus/ethash" "github.com/tomochain/tomochain/core" + "github.com/tomochain/tomochain/core/rawdb" "github.com/tomochain/tomochain/core/types" "github.com/tomochain/tomochain/crypto" "github.com/tomochain/tomochain/params" @@ -38,7 +38,7 @@ var ( testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") testAddress = crypto.PubkeyToAddress(testKey.PublicKey) genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000)) - unknownBlock = types.NewBlock(&types.Header{GasLimit: params.GenesisGasLimit}, nil, nil, nil) + unknownBlock = types.NewBlock(&types.Header{Root: types.EmptyRootHash, GasLimit: params.GenesisGasLimit, BaseFee: big.NewInt(params.InitialBaseFee)}, nil, nil, nil) ) // makeChain creates a chain of n blocks starting at and including parent. @@ -59,7 +59,7 @@ func makeChain(n int, seed byte, parent *types.Block) ([]common.Hash, map[common block.AddTx(tx) } // If the block number is a multiple of 5, add a bonus uncle to the block - if i%5 == 0 { + if i > 0 && i%5 == 0 { block.AddUncle(&types.Header{ParentHash: block.PrevBlock(i - 1).Hash(), Number: big.NewInt(int64(i - 1))}) } }) From 4f0d9bd87157d7cd7a05cb5eeae8c158ae499231 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Thu, 13 Jul 2023 00:21:25 +0700 Subject: [PATCH 79/82] Expose StoredReceiptRLP struct --- core/database_util.go | 11 +---------- core/types/log.go | 37 ------------------------------------- core/types/receipt.go | 6 +++--- 3 files changed, 4 insertions(+), 50 deletions(-) diff --git a/core/database_util.go b/core/database_util.go index 4272ea6340..fda6c5e8c9 100644 --- a/core/database_util.go +++ b/core/database_util.go @@ -224,15 +224,6 @@ func GetTd(db DatabaseReader, hash common.Hash, number uint64) *big.Int { return td } -// storedReceiptRLP is the storage encoding of a receipt. -// Re-definition in core/types/receipt.go. -// TODO: Re-use the existing definition. -type storedReceiptRLP struct { - PostStateOrStatus []byte - CumulativeGasUsed uint64 - Logs []*types.Log -} - // ReceiptLogs is a barebone version of ReceiptForStorage which only keeps // the list of logs. When decoding a stored receipt into this object we // avoid creating the bloom filter. @@ -242,7 +233,7 @@ type receiptLogs struct { // DecodeRLP implements rlp.Decoder. func (r *receiptLogs) DecodeRLP(s *rlp.Stream) error { - var stored storedReceiptRLP + var stored types.StoredReceiptRLP if err := s.Decode(&stored); err != nil { return err } diff --git a/core/types/log.go b/core/types/log.go index 77e79f24a3..a6694cf3b2 100644 --- a/core/types/log.go +++ b/core/types/log.go @@ -102,40 +102,3 @@ func (l *Log) DecodeRLP(s *rlp.Stream) error { func (l *Log) String() string { return fmt.Sprintf(`log: %x %x %x %x %d %x %d`, l.Address, l.Topics, l.Data, l.TxHash, l.TxIndex, l.BlockHash, l.Index) } - -// LogForStorage is a wrapper around a Log that flattens and parses the entire content of -// a log including non-consensus fields. -type LogForStorage Log - -// EncodeRLP implements rlp.Encoder. -func (l *LogForStorage) EncodeRLP(w io.Writer) error { - return rlp.Encode(w, rlpStorageLog{ - Address: l.Address, - Topics: l.Topics, - Data: l.Data, - BlockNumber: l.BlockNumber, - TxHash: l.TxHash, - TxIndex: l.TxIndex, - BlockHash: l.BlockHash, - Index: l.Index, - }) -} - -// DecodeRLP implements rlp.Decoder. -func (l *LogForStorage) DecodeRLP(s *rlp.Stream) error { - var dec rlpStorageLog - err := s.Decode(&dec) - if err == nil { - *l = LogForStorage{ - Address: dec.Address, - Topics: dec.Topics, - Data: dec.Data, - BlockNumber: dec.BlockNumber, - TxHash: dec.TxHash, - TxIndex: dec.TxIndex, - BlockHash: dec.BlockHash, - Index: dec.Index, - } - } - return err -} diff --git a/core/types/receipt.go b/core/types/receipt.go index 45e6feeb18..484324215d 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -90,8 +90,8 @@ type receiptRLP struct { Logs []*Log } -// storedReceiptRLP is the storage encoding of a receipt. -type storedReceiptRLP struct { +// StoredReceiptRLP is the storage encoding of a receipt. +type StoredReceiptRLP struct { PostStateOrStatus []byte CumulativeGasUsed uint64 Logs []*Log @@ -275,7 +275,7 @@ func (r *ReceiptForStorage) EncodeRLP(_w io.Writer) error { // DecodeRLP implements rlp.Decoder, and loads both consensus and implementation // fields of a receipt from an RLP stream. func (r *ReceiptForStorage) DecodeRLP(s *rlp.Stream) error { - var stored storedReceiptRLP + var stored StoredReceiptRLP if err := s.Decode(&stored); err != nil { return err } From 71341a06c806a9a960acce4e1b19698b7e51ccd6 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Thu, 13 Jul 2023 10:36:34 +0700 Subject: [PATCH 80/82] Fix backwards compatibility with receipts --- core/types/receipt.go | 47 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/core/types/receipt.go b/core/types/receipt.go index 484324215d..c966085c34 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -97,6 +97,16 @@ type StoredReceiptRLP struct { Logs []*Log } +type legacyStoredReceiptRLP struct { + PostStateOrStatus []byte + CumulativeGasUsed uint64 + Bloom Bloom + TxHash common.Hash + ContractAddress common.Address + Logs []*Log + GasUsed uint64 +} + // NewReceipt creates a barebone transaction receipt, copying the init fields. func NewReceipt(root []byte, failed bool, cumulativeGasUsed uint64) *Receipt { r := &Receipt{PostState: common.CopyBytes(root), CumulativeGasUsed: cumulativeGasUsed} @@ -275,14 +285,49 @@ func (r *ReceiptForStorage) EncodeRLP(_w io.Writer) error { // DecodeRLP implements rlp.Decoder, and loads both consensus and implementation // fields of a receipt from an RLP stream. func (r *ReceiptForStorage) DecodeRLP(s *rlp.Stream) error { + // Retrieve the entire receipt blob as we need to try multiple decoders + blob, err := s.Raw() + if err != nil { + return err + } + + // Try decoding from the newest format for future proofness, then the older one + // for old nodes that just upgraded. V4 was an intermediate unreleased format so + // we do need to decode it, but it's not common (try last). + if err := decodeStoredReceiptRLP(r, blob); err == nil { + return nil + } + + return decodeLegacyStoredReceiptRLP(r, blob) +} + +func decodeStoredReceiptRLP(r *ReceiptForStorage, blob []byte) error { var stored StoredReceiptRLP - if err := s.Decode(&stored); err != nil { + if err := rlp.DecodeBytes(blob, &stored); err != nil { + return err + } + if err := (*Receipt)(r).setStatus(stored.PostStateOrStatus); err != nil { + return err + } + r.CumulativeGasUsed = stored.CumulativeGasUsed + r.Logs = stored.Logs + r.Bloom = CreateBloom(Receipts{(*Receipt)(r)}) + + return nil +} + +func decodeLegacyStoredReceiptRLP(r *ReceiptForStorage, blob []byte) error { + var stored legacyStoredReceiptRLP + if err := rlp.DecodeBytes(blob, &stored); err != nil { return err } if err := (*Receipt)(r).setStatus(stored.PostStateOrStatus); err != nil { return err } r.CumulativeGasUsed = stored.CumulativeGasUsed + r.TxHash = stored.TxHash + r.ContractAddress = stored.ContractAddress + r.GasUsed = stored.GasUsed r.Logs = stored.Logs r.Bloom = CreateBloom(Receipts{(*Receipt)(r)}) From b5ff4fc1f7667010afa2a59dffaddba6a0fe2d20 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Thu, 13 Jul 2023 10:42:26 +0700 Subject: [PATCH 81/82] Decode legacy receipts with legacy LogForStorage struct --- core/types/log.go | 37 +++++++++++++++++++++++++++++++++++++ core/types/receipt.go | 7 +++++-- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/core/types/log.go b/core/types/log.go index a6694cf3b2..77e79f24a3 100644 --- a/core/types/log.go +++ b/core/types/log.go @@ -102,3 +102,40 @@ func (l *Log) DecodeRLP(s *rlp.Stream) error { func (l *Log) String() string { return fmt.Sprintf(`log: %x %x %x %x %d %x %d`, l.Address, l.Topics, l.Data, l.TxHash, l.TxIndex, l.BlockHash, l.Index) } + +// LogForStorage is a wrapper around a Log that flattens and parses the entire content of +// a log including non-consensus fields. +type LogForStorage Log + +// EncodeRLP implements rlp.Encoder. +func (l *LogForStorage) EncodeRLP(w io.Writer) error { + return rlp.Encode(w, rlpStorageLog{ + Address: l.Address, + Topics: l.Topics, + Data: l.Data, + BlockNumber: l.BlockNumber, + TxHash: l.TxHash, + TxIndex: l.TxIndex, + BlockHash: l.BlockHash, + Index: l.Index, + }) +} + +// DecodeRLP implements rlp.Decoder. +func (l *LogForStorage) DecodeRLP(s *rlp.Stream) error { + var dec rlpStorageLog + err := s.Decode(&dec) + if err == nil { + *l = LogForStorage{ + Address: dec.Address, + Topics: dec.Topics, + Data: dec.Data, + BlockNumber: dec.BlockNumber, + TxHash: dec.TxHash, + TxIndex: dec.TxIndex, + BlockHash: dec.BlockHash, + Index: dec.Index, + } + } + return err +} diff --git a/core/types/receipt.go b/core/types/receipt.go index c966085c34..6d4b1f1bd6 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -103,7 +103,7 @@ type legacyStoredReceiptRLP struct { Bloom Bloom TxHash common.Hash ContractAddress common.Address - Logs []*Log + Logs []*LogForStorage GasUsed uint64 } @@ -328,7 +328,10 @@ func decodeLegacyStoredReceiptRLP(r *ReceiptForStorage, blob []byte) error { r.TxHash = stored.TxHash r.ContractAddress = stored.ContractAddress r.GasUsed = stored.GasUsed - r.Logs = stored.Logs + r.Logs = make([]*Log, len(stored.Logs)) + for i, log := range stored.Logs { + r.Logs[i] = (*Log)(log) + } r.Bloom = CreateBloom(Receipts{(*Receipt)(r)}) return nil From 6859a4f0f24d539029d251029bcf2d1d592235e7 Mon Sep 17 00:00:00 2001 From: Dang Nhat Trinh Date: Thu, 13 Jul 2023 13:37:59 +0700 Subject: [PATCH 82/82] Use the correct chainConfig when getEVM --- accounts/abi/bind/backends/simulated.go | 2 +- core/blockchain.go | 5 +++ core/vm/eips.go | 1 + core/vm/interpreter.go | 52 ++++++++++++------------- eth/api_backend.go | 7 +++- internal/ethapi/api.go | 2 +- internal/ethapi/backend.go | 2 +- les/api_backend.go | 7 +++- miner/worker.go | 3 +- params/config.go | 7 ++-- tests/init.go | 10 +++++ 11 files changed, 59 insertions(+), 39 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index fe68fa8c7e..b95eb34ead 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -342,7 +342,7 @@ func (b *SimulatedBackend) CallContractWithState(call tomochain.CallMsg, chain c evmContext := core.NewEVMContext(msg, chain.CurrentHeader(), chain, nil) // Create a new environment which holds all relevant information // about the transaction and calling mechanisms. - vmenv := vm.NewEVM(evmContext, statedb, nil, chain.Config(), vm.Config{}) + vmenv := vm.NewEVM(evmContext, statedb, nil, chain.Config(), *b.blockchain.GetVMConfig()) gaspool := new(core.GasPool).AddGas(1000000) owner := common.Address{} rval, _, _, err := core.NewStateTransition(vmenv, msg, gaspool).TransitionDb(owner) diff --git a/core/blockchain.go b/core/blockchain.go index 1186ea2ac3..42a5cb3cf6 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -699,6 +699,11 @@ func (bc *BlockChain) Genesis() *types.Block { return bc.genesisBlock } +// GetVMConfig returns the blockchain VM config. +func (bc *BlockChain) GetVMConfig() *vm.Config { + return &bc.vmConfig +} + // GetBody retrieves a block body (transactions and uncles) from the database by // hash, caching it if found. func (bc *BlockChain) GetBody(hash common.Hash) *types.Body { diff --git a/core/vm/eips.go b/core/vm/eips.go index 46d2c0727b..72ccda5e38 100644 --- a/core/vm/eips.go +++ b/core/vm/eips.go @@ -103,6 +103,7 @@ func enable3198(jt *JumpTable) { constantGas: GasQuickStep, minStack: minStack(0, 1), maxStack: maxStack(0, 1), + valid: true, } } diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 3c97467150..4c73193e2d 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -98,35 +98,33 @@ func NewEVMInterpreter(evm *EVM, cfg Config) *EVMInterpreter { // We use the STOP instruction whether to see // the jump table was initialised. If it was not // we'll set the default jump table. - if !cfg.JumpTable[STOP].valid { - var jt JumpTable - switch { - case evm.chainRules.IsLondon: - jt = londonInstructionSet - case evm.chainRules.IsIstanbul: - jt = istanbulInstructionSet - case evm.chainRules.IsConstantinople: - jt = constantinopleInstructionSet - case evm.chainRules.IsByzantium: - jt = byzantiumInstructionSet - case evm.chainRules.IsEIP158: - jt = spuriousDragonInstructionSet - case evm.chainRules.IsEIP150: - jt = tangerineWhistleInstructionSet - case evm.chainRules.IsHomestead: - jt = homesteadInstructionSet - default: - jt = frontierInstructionSet - } - for i, eip := range cfg.ExtraEips { - if err := EnableEIP(eip, &jt); err != nil { - // Disable it, so caller can check if it's activated or not - cfg.ExtraEips = append(cfg.ExtraEips[:i], cfg.ExtraEips[i+1:]...) - log.Error("EIP activation failed", "eip", eip, "error", err) - } + var jt JumpTable + switch { + case evm.chainRules.IsLondon: + jt = londonInstructionSet + case evm.chainRules.IsIstanbul: + jt = istanbulInstructionSet + case evm.chainRules.IsConstantinople: + jt = constantinopleInstructionSet + case evm.chainRules.IsByzantium: + jt = byzantiumInstructionSet + case evm.chainRules.IsEIP158: + jt = spuriousDragonInstructionSet + case evm.chainRules.IsEIP150: + jt = tangerineWhistleInstructionSet + case evm.chainRules.IsHomestead: + jt = homesteadInstructionSet + default: + jt = frontierInstructionSet + } + for i, eip := range cfg.ExtraEips { + if err := EnableEIP(eip, &jt); err != nil { + // Disable it, so caller can check if it's activated or not + cfg.ExtraEips = append(cfg.ExtraEips[:i], cfg.ExtraEips[i+1:]...) + log.Error("EIP activation failed", "eip", eip, "error", err) } - cfg.JumpTable = jt } + cfg.JumpTable = jt return &EVMInterpreter{ evm: evm, diff --git a/eth/api_backend.go b/eth/api_backend.go index 815b1e22e4..fb2964abbc 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -220,12 +220,15 @@ func (b *EthApiBackend) GetTd(blockHash common.Hash) *big.Int { return b.eth.blockchain.GetTdByHash(blockHash) } -func (b *EthApiBackend) GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, tomoxState *tradingstate.TradingStateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) { +func (b *EthApiBackend) GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, tomoxState *tradingstate.TradingStateDB, header *types.Header, vmCfg *vm.Config) (*vm.EVM, func() error, error) { + if vmCfg == nil { + vmCfg = b.eth.blockchain.GetVMConfig() + } state.SetBalance(msg.From, math.MaxBig256) vmError := func() error { return nil } context := core.NewEVMContext(msg, header, b.eth.BlockChain(), nil) - return vm.NewEVM(context, state, tomoxState, b.eth.chainConfig, vmCfg), vmError, nil + return vm.NewEVM(context, state, tomoxState, b.eth.chainConfig, *vmCfg), vmError, nil } func (b *EthApiBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 0b7e8e6979..413062e45a 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1108,7 +1108,7 @@ func (s *PublicBlockChainAPI) doCall(ctx context.Context, args TransactionArgs, return nil, 0, false, err } // Get a new instance of the EVM. - evm, vmError, err := s.b.GetEVM(ctx, msg, statedb, tomoxState, header, vm.Config{NoBaseFee: true}) + evm, vmError, err := s.b.GetEVM(ctx, msg, statedb, tomoxState, header, &vm.Config{NoBaseFee: true}) if err != nil { return nil, 0, false, err } diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index 9a88c2310b..7c44b34b1f 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -68,7 +68,7 @@ type Backend interface { GetBlock(ctx context.Context, blockHash common.Hash) (*types.Block, error) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) GetTd(blockHash common.Hash) *big.Int - GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, tomoxState *tradingstate.TradingStateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) + GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, tomoxState *tradingstate.TradingStateDB, header *types.Header, vmCfg *vm.Config) (*vm.EVM, func() error, error) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription diff --git a/les/api_backend.go b/les/api_backend.go index cafeb5858f..f72e861a6a 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -154,10 +154,13 @@ func (b *LesApiBackend) GetTd(blockHash common.Hash) *big.Int { return b.eth.blockchain.GetTdByHash(blockHash) } -func (b *LesApiBackend) GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, tomoxState *tradingstate.TradingStateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) { +func (b *LesApiBackend) GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, tomoxState *tradingstate.TradingStateDB, header *types.Header, vmCfg *vm.Config) (*vm.EVM, func() error, error) { + if vmCfg == nil { + vmCfg = new(vm.Config) + } state.SetBalance(msg.From, math.MaxBig256) context := core.NewEVMContext(msg, header, b.eth.blockchain, nil) - return vm.NewEVM(context, state, tomoxState, b.eth.chainConfig, vmCfg), state.Error, nil + return vm.NewEVM(context, state, tomoxState, b.eth.chainConfig, *vmCfg), state.Error, nil } func (b *LesApiBackend) SendTx(ctx context.Context, signedTx *types.Transaction) error { diff --git a/miner/worker.go b/miner/worker.go index d2b0c991d8..e0773cbbf0 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -41,7 +41,6 @@ import ( "github.com/tomochain/tomochain/core" "github.com/tomochain/tomochain/core/state" "github.com/tomochain/tomochain/core/types" - "github.com/tomochain/tomochain/core/vm" "github.com/tomochain/tomochain/ethdb" "github.com/tomochain/tomochain/event" "github.com/tomochain/tomochain/log" @@ -1091,7 +1090,7 @@ func (env *Work) commitTransactions(mux *event.TypeMux, balanceFee map[common.Ad func (env *Work) commitTransaction(balanceFee map[common.Address]*big.Int, tx *types.Transaction, bc *core.BlockChain, coinbase common.Address, gp *core.GasPool) (error, []*types.Log, bool, uint64) { snap := env.state.Snapshot() - receipt, gas, err, tokenFeeUsed := core.ApplyTransaction(env.config, balanceFee, bc, &coinbase, gp, env.state, env.tradingState, env.header, tx, &env.header.GasUsed, vm.Config{}) + receipt, gas, err, tokenFeeUsed := core.ApplyTransaction(env.config, balanceFee, bc, &coinbase, gp, env.state, env.tradingState, env.header, tx, &env.header.GasUsed, *bc.GetVMConfig()) if err != nil { env.state.RevertToSnapshot(snap) return err, nil, false, 0 diff --git a/params/config.go b/params/config.go index 4ca56255d8..52bba84eae 100644 --- a/params/config.go +++ b/params/config.go @@ -39,6 +39,7 @@ var ( EIP155Block: big.NewInt(3), EIP158Block: big.NewInt(3), ByzantiumBlock: big.NewInt(4), + LondonBlock: big.NewInt(5), Posv: &PosvConfig{ Period: 2, Epoch: 900, @@ -109,8 +110,8 @@ var ( // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllPosvProtocolChanges = &ChainConfig{big.NewInt(89), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, &PosvConfig{Period: 0, Epoch: 30000}} - AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}, nil} + AllPosvProtocolChanges = &ChainConfig{big.NewInt(89), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, big.NewInt(0), nil, nil, &PosvConfig{Period: 0, Epoch: 30000}} + AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, big.NewInt(0), nil, &CliqueConfig{Period: 0, Epoch: 30000}, nil} TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, big.NewInt(0), new(EthashConfig), nil, nil} TestRules = TestChainConfig.Rules(new(big.Int)) ) @@ -137,7 +138,7 @@ type ChainConfig struct { ByzantiumBlock *big.Int `json:"byzantiumBlock,omitempty"` // Byzantium switch block (nil = no fork, 0 = already on byzantium) ConstantinopleBlock *big.Int `json:"constantinopleBlock,omitempty"` // Constantinople switch block (nil = no fork, 0 = already activated) - LondonBlock *big.Int `json:"londonBlock,omitempty"` // Berlin switch block (nil = no fork, 0 = already on berlin) + LondonBlock *big.Int `json:"londonBlock,omitempty"` // London switch block (nil = no fork, 0 = already on london) // Various consensus engines Ethash *EthashConfig `json:"ethash,omitempty"` diff --git a/tests/init.go b/tests/init.go index de6bb71cf4..216e5e12e4 100644 --- a/tests/init.go +++ b/tests/init.go @@ -76,6 +76,16 @@ var Forks = map[string]*params.ChainConfig{ EIP158Block: big.NewInt(0), ByzantiumBlock: big.NewInt(5), }, + "London": { + ChainId: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + }, } // UnsupportedForkError is returned when a test requests a fork that isn't implemented.