From ae6c52ce7c90a29414468082acafdb050deb1834 Mon Sep 17 00:00:00 2001 From: divfor Date: Sun, 18 Oct 2020 00:57:25 +0800 Subject: [PATCH 01/14] Add multi source IP support fixes #476 --- cmd/options.go | 12 +++ js/runner.go | 5 +- lib/options.go | 6 ++ lib/options_test.go | 17 ++++ lib/types/ipblock.go | 173 ++++++++++++++++++++++++++++++++++++++ lib/types/ipblock_test.go | 99 ++++++++++++++++++++++ 6 files changed, 311 insertions(+), 1 deletion(-) create mode 100644 lib/types/ipblock.go create mode 100644 lib/types/ipblock_test.go diff --git a/cmd/options.go b/cmd/options.go index 0276b4daaab..530290d6690 100644 --- a/cmd/options.go +++ b/cmd/options.go @@ -93,6 +93,7 @@ func optionFlagSet() *pflag.FlagSet { flags.StringSlice("tag", nil, "add a `tag` to be applied to all samples, as `[name]=[value]`") flags.String("console-output", "", "redirects the console logging to the provided output file") flags.Bool("discard-response-bodies", false, "Read but don't process or save HTTP response bodies") + flags.String("client-ips", "", "Client IP Ranges and/or CIDRs from which each VU will be making requests") flags.String("dns", types.DefaultDNSConfig().String(), "DNS resolver configuration. Possible ttl values are: 'inf' "+ "for a persistent cache, '0' to disable the cache,\nor a positive duration, e.g. '1s', '1m', etc. "+ "Milliseconds are assumed if no unit is provided.\n"+ @@ -205,6 +206,17 @@ func getOptions(flags *pflag.FlagSet) (lib.Options, error) { } } + ipRangesString, err := flags.GetString("client-ips") + if err != nil { + return opts, err + } + if flags.Changed("client-ips") { + err = opts.ClientIPRanges.UnmarshalText([]byte(ipRangesString)) + if err != nil { + return opts, err + } + } + if flags.Changed("summary-trend-stats") { trendStats, errSts := flags.GetStringSlice("summary-trend-stats") if errSts != nil { diff --git a/js/runner.go b/js/runner.go index 370ebf17790..bc404c6d3b4 100644 --- a/js/runner.go +++ b/js/runner.go @@ -170,6 +170,10 @@ func (r *Runner) newVU(id int64, samplesOut chan<- stats.SampleContainer) (*VU, BlockedHostnames: r.Bundle.Options.BlockedHostnames.Trie, Hosts: r.Bundle.Options.Hosts, } + if r.Bundle.Options.ClientIPRanges.Valid { + dialer.Dialer.LocalAddr = &net.TCPAddr{IP: r.Bundle.Options.ClientIPRanges.Pool.GetIP(uint64(id - 1))} + } + tlsConfig := &tls.Config{ InsecureSkipVerify: r.Bundle.Options.InsecureSkipTLSVerify.Bool, CipherSuites: cipherSuites, @@ -307,7 +311,6 @@ func (r *Runner) IsExecutable(name string) bool { func (r *Runner) SetOptions(opts lib.Options) error { r.Bundle.Options = opts - r.RPSLimit = nil if rps := opts.RPS; rps.Valid { r.RPSLimit = rate.NewLimiter(rate.Limit(rps.Int64), 1) diff --git a/lib/options.go b/lib/options.go index 6dfec6ed4c7..85e422e0477 100644 --- a/lib/options.go +++ b/lib/options.go @@ -388,6 +388,9 @@ type Options struct { // Redirect console logging to a file ConsoleOutput null.String `json:"-" envconfig:"K6_CONSOLE_OUTPUT"` + + // Specify client IP ranges and/or CIDR from which VUs will make requests + ClientIPRanges types.NullIPPool `json:"-" envconfig:"K6_CLIENT_IPS"` } // Returns the result of overwriting any fields with any that are set on the argument. @@ -542,6 +545,9 @@ func (o Options) Apply(opts Options) Options { if opts.ConsoleOutput.Valid { o.ConsoleOutput = opts.ConsoleOutput } + if opts.ClientIPRanges.Valid { + o.ClientIPRanges = opts.ClientIPRanges + } if opts.DNS.TTL.Valid { o.DNS.TTL = opts.DNS.TTL } diff --git a/lib/options_test.go b/lib/options_test.go index a7eb55b4147..91f7bb8cc69 100644 --- a/lib/options_test.go +++ b/lib/options_test.go @@ -407,9 +407,21 @@ func TestOptions(t *testing.T) { assert.True(t, opts.DiscardResponseBodies.Valid) assert.True(t, opts.DiscardResponseBodies.Bool) }) + t.Run("ClientIPRanges", func(t *testing.T) { + clientIPRanges, err := types.NewIPPool("129.112.232.12,123.12.0.0/32") + require.NoError(t, err) + opts := Options{}.Apply(Options{ClientIPRanges: types.NullIPPool{Pool: clientIPRanges, Valid: true}}) + assert.NotNil(t, opts.ClientIPRanges) + }) } func TestOptionsEnv(t *testing.T) { + mustIPPool := func(s string) *types.IPPool { + p, err := types.NewIPPool(s) + require.NoError(t, err) + return p + } + testdata := map[struct{ Name, Key string }]map[string]interface{}{ {"Paused", "K6_PAUSED"}: { "": null.Bool{}, @@ -469,6 +481,11 @@ func TestOptionsEnv(t *testing.T) { "": null.String{}, "Hi!": null.StringFrom("Hi!"), }, + {"ClientIPRanges", "K6_CLIENT_IPS"}: { + "": types.NullIPPool{}, + "192.168.220.2": types.NullIPPool{Pool: mustIPPool("192.168.220.2"), Valid: true}, + "192.168.220.2/24": types.NullIPPool{Pool: mustIPPool("192.168.220.0/24"), Valid: true}, + }, {"Throw", "K6_THROW"}: { "": null.Bool{}, "true": null.BoolFrom(true), diff --git a/lib/types/ipblock.go b/lib/types/ipblock.go new file mode 100644 index 00000000000..f02b9fcb47f --- /dev/null +++ b/lib/types/ipblock.go @@ -0,0 +1,173 @@ +package types + +import ( + "errors" + "fmt" + "math/big" + "net" + "strings" +) + +// ipBlock represents a continuous segment of IP addresses +type ipBlock struct { + // TODO rename count as it is actually the accumulation of the counts up to and including this + // on in the pool + // maybe add another field, although technically we need count only to find out when to enter this + // block not for anything else + firstIP, count *big.Int + ipv6 bool +} + +// IPPool represent a slice of IPBlocks +type IPPool struct { + list []ipBlock + count *big.Int +} + +func getIPBlock(s string) (*ipBlock, error) { + switch { + case strings.Contains(s, "-"): + return ipBlockFromRange(s) + case strings.Contains(s, "/"): + return ipBlockFromCIDR(s) + default: + if net.ParseIP(s) == nil { + return nil, fmt.Errorf("%s is not a valid ip, range ip or CIDR", s) + } + return ipBlockFromRange(s + "-" + s) + } +} + +func ipBlockFromRange(s string) (*ipBlock, error) { + ss := strings.SplitN(s, "-", 2) + ip0Str, ip1Str := strings.TrimSpace(ss[0]), strings.TrimSpace(ss[1]) + ip0, ip1 := net.ParseIP(ip0Str), net.ParseIP(ip1Str) + if ip0 == nil || ip1 == nil { + fmt.Println("Wrong IP range format: ", s) + return nil, errors.New("wrong IP range format: " + s) + } + if (ip0.To4() == nil) != (ip1.To4() == nil) { // XOR + return nil, errors.New("mixed IP range format: " + s) + } + block := ipBlockFromTwoIPs(ip0, ip1) + + if block.count.Sign() <= 0 { + return nil, errors.New("negative IP range: " + s) + } + return block, nil +} + +func ipBlockFromTwoIPs(ip0, ip1 net.IP) *ipBlock { + var block ipBlock + block.firstIP = new(big.Int) + block.count = new(big.Int) + block.ipv6 = ip0.To4() == nil + if block.ipv6 { + block.firstIP.SetBytes(ip0.To16()) + block.count.SetBytes(ip1.To16()) + } else { + block.firstIP.SetBytes(ip0.To4()) + block.count.SetBytes(ip1.To4()) + } + block.count.Sub(block.count, block.firstIP) + block.count.Add(block.count, big.NewInt(1)) + + return &block +} + +func ipBlockFromCIDR(s string) (*ipBlock, error) { + _, pnet, err := net.ParseCIDR(s) // range start ip, cidr ipnet + if err != nil { + fmt.Println("ParseCIDR() failed: ", s) + return nil, fmt.Errorf("parseCIDR() failed parsing %s: %w", s, err) + } + ip0 := pnet.IP + // TODO: this is just to copy it, it will probably be better to copy the bytes ... + ip1 := net.ParseIP(ip0.String()) + if ip1.To4() == nil { + ip1 = ip1.To16() + } else { + ip1 = ip1.To4() + } + for i := range ip1 { + ip1[i] |= (255 ^ pnet.Mask[i]) + } + block := ipBlockFromTwoIPs(ip0, ip1) + // in the case of ipv4 if the network is bigger than 31 the first and last IP are reserved so we + // need to reduce the addresses by 2 and increment the first ip + if !block.ipv6 && big.NewInt(2).Cmp(block.count) < 0 { + block.count.Sub(block.count, big.NewInt(2)) + block.firstIP.Add(block.firstIP, big.NewInt(1)) + } + return block, nil +} + +func (b ipBlock) getIP(index *big.Int) net.IP { + // TODO implement walking ipv6 networks first + // that will probably require more math ... including knowing which is the next network and ... + // thinking about it - it looks like it's going to be kind of hard or badly defined + i := new(big.Int) + i.Add(b.firstIP, index) + // TODO use big.Int.FillBytes when golang 1.14 is no longer supported + return net.IP(i.Bytes()) +} + +// NewIPPool return an IPBlock slice with corret weight and mode +// Possible format is range1[:mode[:weight]][,range2[:mode[:weight]]] +func NewIPPool(ranges string) (*IPPool, error) { + ss := strings.Split(strings.TrimSpace(ranges), ",") + pool := &IPPool{} + pool.list = make([]ipBlock, len(ss)) + pool.count = new(big.Int) + for i, bs := range ss { + r, err := getIPBlock(bs) + if err != nil { + return nil, err + } + + pool.count.Add(pool.count, r.count) + r.count.Set(pool.count) + pool.list[i] = *r + } + return pool, nil +} + +// GetIP return an IP from a pool of IPBlock slice +func (pool *IPPool) GetIP(id uint64) net.IP { + return pool.GetIPBig(new(big.Int).SetUint64(id)) +} + +// GetIPBig returns an IP form the pool with the provided id that is big.Int +func (pool *IPPool) GetIPBig(id *big.Int) net.IP { + id = new(big.Int).Rem(id, pool.count) + for i, b := range pool.list { + if id.Cmp(b.count) < 0 { + if i > 0 { + id.Sub(id, pool.list[i-1].count) + } + return b.getIP(id) + } + } + return nil +} + +// NullIPPool is a nullable IPPool +type NullIPPool struct { + Pool *IPPool + Valid bool +} + +// UnmarshalText converts text data to a valid NullIPPool +func (n *NullIPPool) UnmarshalText(data []byte) error { + if len(data) == 0 { + *n = NullIPPool{} + return nil + } + var err error + n.Pool, err = NewIPPool(string(data)) + if err != nil { + return err + } + n.Valid = true + return nil +} diff --git a/lib/types/ipblock_test.go b/lib/types/ipblock_test.go new file mode 100644 index 00000000000..18ea8a14e82 --- /dev/null +++ b/lib/types/ipblock_test.go @@ -0,0 +1,99 @@ +package types + +import ( + "math/big" + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +//nolint:gochecknoglobals +var max64 = new(big.Int).Exp(big.NewInt(2), big.NewInt(64), nil) + +func get128BigInt(hi, lo int64) *big.Int { + h := big.NewInt(hi) + h.Mul(h, max64) + return h.Add(h, big.NewInt(lo)) +} + +func TestIpBlock(t *testing.T) { + testdata := map[string]struct { + count *big.Int + firstIP, lastIP net.IP + }{ + "192.168.0.101": {new(big.Int).SetInt64(1), net.ParseIP("192.168.0.101"), net.ParseIP("192.168.0.101")}, + + "192.168.0.101-192.168.0.200": {new(big.Int).SetInt64(100), net.ParseIP("192.168.0.101"), net.ParseIP("192.168.0.200")}, + "192.168.0.100-192.168.0.200": {new(big.Int).SetInt64(101), net.ParseIP("192.168.0.100"), net.ParseIP("192.168.0.200")}, + "fd00:1:1:0::0-fd00:1:1:ff::3ff": {get128BigInt(255, 1024), net.ParseIP("fd00:1:1:0::0"), net.ParseIP("fd00:1:1:ff::3ff")}, + "fd00:1:1:2::1-fd00:1:1:ff::3ff": {get128BigInt(253, 1023), net.ParseIP("fd00:1:1:2::1"), net.ParseIP("fd00:1:1:ff::3ff")}, + + "192.168.0.0/16": {get128BigInt(0, 65534), net.ParseIP("192.168.0.1"), net.ParseIP("192.168.255.254")}, + "192.168.0.1/16": {get128BigInt(0, 65534), net.ParseIP("192.168.0.1"), net.ParseIP("192.168.255.254")}, + "192.168.0.10/16": {get128BigInt(0, 65534), net.ParseIP("192.168.0.1"), net.ParseIP("192.168.255.254")}, + "192.168.0.10/31": {get128BigInt(0, 2), net.ParseIP("192.168.0.10"), net.ParseIP("192.168.0.11")}, + "192.168.0.10/32": {get128BigInt(0, 1), net.ParseIP("192.168.0.10"), net.ParseIP("192.168.0.10")}, + "fd00::0/120": {get128BigInt(0, 256), net.ParseIP("fd00::0"), net.ParseIP("fd00::ff")}, + "fd00::1/120": {get128BigInt(0, 256), net.ParseIP("fd00::0"), net.ParseIP("fd00::ff")}, + "fd00::3/120": {get128BigInt(0, 256), net.ParseIP("fd00::0"), net.ParseIP("fd00::ff")}, + "fd00::0/112": {get128BigInt(0, 65536), net.ParseIP("fd00::0"), net.ParseIP("fd00::ffff")}, + "fd00::1/112": {get128BigInt(0, 65536), net.ParseIP("fd00::0"), net.ParseIP("fd00::ffff")}, + "fd00::2/112": {get128BigInt(0, 65536), net.ParseIP("fd00::0"), net.ParseIP("fd00::ffff")}, + } + for name, data := range testdata { + name, data := name, data + t.Run(name, func(t *testing.T) { + b, err := getIPBlock(name) + require.NoError(t, err) + assert.Equal(t, data.count, b.count) + idx := big.NewInt(0) + assert.Equal(t, data.firstIP.To16(), b.getIP(idx).To16()) + idx.Sub(idx.Add(idx, b.count), big.NewInt(1)) + assert.Equal(t, data.lastIP.To16(), b.getIP(idx).To16()) + }) + } +} + +func TestIPPool(t *testing.T) { + testdata := map[string]struct { + count *big.Int + queries map[uint64]net.IP + }{ + "192.168.0.101": { + count: new(big.Int).SetInt64(1), + queries: map[uint64]net.IP{0: net.ParseIP("192.168.0.101"), 12: net.ParseIP("192.168.0.101")}, + }, + "192.168.0.101,192.168.0.102": { + count: new(big.Int).SetInt64(2), + queries: map[uint64]net.IP{ + 0: net.ParseIP("192.168.0.101"), + 1: net.ParseIP("192.168.0.102"), + 12: net.ParseIP("192.168.0.101"), + 13: net.ParseIP("192.168.0.102"), + }, + }, + "192.168.0.101-192.168.0.105,fd00::2/112": { + count: new(big.Int).SetInt64(65541), + queries: map[uint64]net.IP{ + 0: net.ParseIP("192.168.0.101"), + 1: net.ParseIP("192.168.0.102"), + 5: net.ParseIP("fd00::0"), + 6: net.ParseIP("fd00::1"), + 65541: net.ParseIP("192.168.0.101"), + }, + }, + } + for name, data := range testdata { + name, data := name, data + t.Run(name, func(t *testing.T) { + p, err := NewIPPool(name) + require.NoError(t, err) + assert.Equal(t, data.count, p.count) + for q, a := range data.queries { + assert.Equal(t, a.To16(), p.GetIP(q).To16(), "index %d", q) + } + }) + } +} From 02531b5e3e185145a63e69042c5c44d1a07914fe Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Mon, 26 Oct 2020 13:15:10 +0200 Subject: [PATCH 02/14] add copyright --- lib/types/ipblock.go | 20 ++++++++++++++++++++ lib/types/ipblock_test.go | 20 ++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/lib/types/ipblock.go b/lib/types/ipblock.go index f02b9fcb47f..ce8e08d0097 100644 --- a/lib/types/ipblock.go +++ b/lib/types/ipblock.go @@ -1,3 +1,23 @@ +/* + * + * k6 - a next-generation load testing tool + * Copyright (C) 2020 Load Impact + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + package types import ( diff --git a/lib/types/ipblock_test.go b/lib/types/ipblock_test.go index 18ea8a14e82..04e7ba7ee03 100644 --- a/lib/types/ipblock_test.go +++ b/lib/types/ipblock_test.go @@ -1,3 +1,23 @@ +/* + * + * k6 - a next-generation load testing tool + * Copyright (C) 2020 Load Impact + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + package types import ( From dc2cf50f9060d966d9c80c89609b9c4372786223 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Mon, 26 Oct 2020 13:17:52 +0200 Subject: [PATCH 03/14] Drop fmt.Pritnln lines --- lib/types/ipblock.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/types/ipblock.go b/lib/types/ipblock.go index ce8e08d0097..a2662764d9c 100644 --- a/lib/types/ipblock.go +++ b/lib/types/ipblock.go @@ -63,7 +63,6 @@ func ipBlockFromRange(s string) (*ipBlock, error) { ip0Str, ip1Str := strings.TrimSpace(ss[0]), strings.TrimSpace(ss[1]) ip0, ip1 := net.ParseIP(ip0Str), net.ParseIP(ip1Str) if ip0 == nil || ip1 == nil { - fmt.Println("Wrong IP range format: ", s) return nil, errors.New("wrong IP range format: " + s) } if (ip0.To4() == nil) != (ip1.To4() == nil) { // XOR @@ -98,7 +97,6 @@ func ipBlockFromTwoIPs(ip0, ip1 net.IP) *ipBlock { func ipBlockFromCIDR(s string) (*ipBlock, error) { _, pnet, err := net.ParseCIDR(s) // range start ip, cidr ipnet if err != nil { - fmt.Println("ParseCIDR() failed: ", s) return nil, fmt.Errorf("parseCIDR() failed parsing %s: %w", s, err) } ip0 := pnet.IP From 01408f7f55e88ae87c0b123a606d482d6b01486e Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Mon, 26 Oct 2020 13:35:52 +0200 Subject: [PATCH 04/14] changes to comments --- lib/types/ipblock.go | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/lib/types/ipblock.go b/lib/types/ipblock.go index a2662764d9c..b485899d6ef 100644 --- a/lib/types/ipblock.go +++ b/lib/types/ipblock.go @@ -52,7 +52,7 @@ func getIPBlock(s string) (*ipBlock, error) { return ipBlockFromCIDR(s) default: if net.ParseIP(s) == nil { - return nil, fmt.Errorf("%s is not a valid ip, range ip or CIDR", s) + return nil, fmt.Errorf("%s is not a valid IP, IP range or CIDR", s) } return ipBlockFromRange(s + "-" + s) } @@ -77,6 +77,8 @@ func ipBlockFromRange(s string) (*ipBlock, error) { } func ipBlockFromTwoIPs(ip0, ip1 net.IP) *ipBlock { + // This code doesn't do any checks on the validity of the arguments, that should be + // done before and/or after it is called var block ipBlock block.firstIP = new(big.Int) block.count = new(big.Int) @@ -95,7 +97,7 @@ func ipBlockFromTwoIPs(ip0, ip1 net.IP) *ipBlock { } func ipBlockFromCIDR(s string) (*ipBlock, error) { - _, pnet, err := net.ParseCIDR(s) // range start ip, cidr ipnet + _, pnet, err := net.ParseCIDR(s) if err != nil { return nil, fmt.Errorf("parseCIDR() failed parsing %s: %w", s, err) } @@ -130,8 +132,8 @@ func (b ipBlock) getIP(index *big.Int) net.IP { return net.IP(i.Bytes()) } -// NewIPPool return an IPBlock slice with corret weight and mode -// Possible format is range1[:mode[:weight]][,range2[:mode[:weight]]] +// NewIPPool returns an IPPool slice from the provided string represenation that should be comma +// separated list of IPs, IP ranges(ip1-ip2) and CIDRs func NewIPPool(ranges string) (*IPPool, error) { ss := strings.Split(strings.TrimSpace(ranges), ",") pool := &IPPool{} @@ -145,25 +147,25 @@ func NewIPPool(ranges string) (*IPPool, error) { pool.count.Add(pool.count, r.count) r.count.Set(pool.count) - pool.list[i] = *r + pool.list[i] = ipPoolBlo } return pool, nil } // GetIP return an IP from a pool of IPBlock slice -func (pool *IPPool) GetIP(id uint64) net.IP { - return pool.GetIPBig(new(big.Int).SetUint64(id)) +func (pool *IPPool) GetIP(index uint64) net.IP { + return pool.GetIPBig(new(big.Int).SetUint64(index)) } -// GetIPBig returns an IP form the pool with the provided id that is big.Int -func (pool *IPPool) GetIPBig(id *big.Int) net.IP { - id = new(big.Int).Rem(id, pool.count) +// GetIPBig returns an IP from the pool with the provided index that is big.Int +func (pool *IPPool) GetIPBig(index *big.Int) net.IP { + index = new(big.Int).Rem(index, pool.count) for i, b := range pool.list { - if id.Cmp(b.count) < 0 { + if index.Cmp(b.count) < 0 { if i > 0 { - id.Sub(id, pool.list[i-1].count) + index.Sub(index, pool.list[i-1].count) } - return b.getIP(id) + return b.getIP(index) } } return nil From ebd5b470e182c2a868d6b91e0fa999aa12973b20 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Mon, 26 Oct 2020 13:52:54 +0200 Subject: [PATCH 05/14] add ipPoolBlock for better readability --- lib/types/ipblock.go | 41 ++++++++++++++++++++++++--------------- lib/types/ipblock_test.go | 5 +++-- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/lib/types/ipblock.go b/lib/types/ipblock.go index b485899d6ef..6d96bb93a1d 100644 --- a/lib/types/ipblock.go +++ b/lib/types/ipblock.go @@ -30,17 +30,19 @@ import ( // ipBlock represents a continuous segment of IP addresses type ipBlock struct { - // TODO rename count as it is actually the accumulation of the counts up to and including this - // on in the pool - // maybe add another field, although technically we need count only to find out when to enter this - // block not for anything else firstIP, count *big.Int ipv6 bool } +// ipPoolBlock is almost the same as ipBlock but is put in a IPPool and knows it's start id indest +// of it's size/count +type ipPoolBlock struct { + firstIP, startIndex *big.Int +} + // IPPool represent a slice of IPBlocks type IPPool struct { - list []ipBlock + list []ipPoolBlock count *big.Int } @@ -122,7 +124,7 @@ func ipBlockFromCIDR(s string) (*ipBlock, error) { return block, nil } -func (b ipBlock) getIP(index *big.Int) net.IP { +func (b ipPoolBlock) getIP(index *big.Int) net.IP { // TODO implement walking ipv6 networks first // that will probably require more math ... including knowing which is the next network and ... // thinking about it - it looks like it's going to be kind of hard or badly defined @@ -132,12 +134,12 @@ func (b ipBlock) getIP(index *big.Int) net.IP { return net.IP(i.Bytes()) } -// NewIPPool returns an IPPool slice from the provided string represenation that should be comma +// NewIPPool returns an IPPool slice from the provided string representation that should be comma // separated list of IPs, IP ranges(ip1-ip2) and CIDRs func NewIPPool(ranges string) (*IPPool, error) { ss := strings.Split(strings.TrimSpace(ranges), ",") pool := &IPPool{} - pool.list = make([]ipBlock, len(ss)) + pool.list = make([]ipPoolBlock, len(ss)) pool.count = new(big.Int) for i, bs := range ss { r, err := getIPBlock(bs) @@ -145,9 +147,18 @@ func NewIPPool(ranges string) (*IPPool, error) { return nil, err } + pool.list[i] = ipPoolBlock{ + firstIP: r.firstIP, + startIndex: new(big.Int).Set(pool.count), // this is how many there are until now + } pool.count.Add(pool.count, r.count) - r.count.Set(pool.count) - pool.list[i] = ipPoolBlo + } + + // The list gets reversed here as later it is searched based on when the index we are looking is + // bigger than startIndex but it will be true always for the first block which is with + // startIndex 0. This can also be fixed by iterating in reverse but this seems better + for i := 0; i < len(pool.list)/2; i++ { + pool.list[i], pool.list[len(pool.list)/2-i] = pool.list[len(pool.list)/2-i], pool.list[i] } return pool, nil } @@ -160,12 +171,10 @@ func (pool *IPPool) GetIP(index uint64) net.IP { // GetIPBig returns an IP from the pool with the provided index that is big.Int func (pool *IPPool) GetIPBig(index *big.Int) net.IP { index = new(big.Int).Rem(index, pool.count) - for i, b := range pool.list { - if index.Cmp(b.count) < 0 { - if i > 0 { - index.Sub(index, pool.list[i-1].count) - } - return b.getIP(index) + for _, b := range pool.list { + fmt.Println(index, b.startIndex) + if index.Cmp(b.startIndex) >= 0 { + return b.getIP(index.Sub(index, b.startIndex)) } } return nil diff --git a/lib/types/ipblock_test.go b/lib/types/ipblock_test.go index 04e7ba7ee03..070c15a2424 100644 --- a/lib/types/ipblock_test.go +++ b/lib/types/ipblock_test.go @@ -68,10 +68,11 @@ func TestIpBlock(t *testing.T) { b, err := getIPBlock(name) require.NoError(t, err) assert.Equal(t, data.count, b.count) + pb := ipPoolBlock{firstIP: b.firstIP} idx := big.NewInt(0) - assert.Equal(t, data.firstIP.To16(), b.getIP(idx).To16()) + assert.Equal(t, data.firstIP.To16(), pb.getIP(idx).To16()) idx.Sub(idx.Add(idx, b.count), big.NewInt(1)) - assert.Equal(t, data.lastIP.To16(), b.getIP(idx).To16()) + assert.Equal(t, data.lastIP.To16(), pb.getIP(idx).To16()) }) } } From 9a3f095e22cbfca63e7738793e9f53ff60c8f284 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Mon, 26 Oct 2020 14:01:55 +0200 Subject: [PATCH 06/14] Rename to local-ips --- cmd/options.go | 8 ++++---- js/runner.go | 4 ++-- lib/options.go | 6 +++--- lib/options_test.go | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cmd/options.go b/cmd/options.go index 530290d6690..81bfbb4b47e 100644 --- a/cmd/options.go +++ b/cmd/options.go @@ -93,7 +93,7 @@ func optionFlagSet() *pflag.FlagSet { flags.StringSlice("tag", nil, "add a `tag` to be applied to all samples, as `[name]=[value]`") flags.String("console-output", "", "redirects the console logging to the provided output file") flags.Bool("discard-response-bodies", false, "Read but don't process or save HTTP response bodies") - flags.String("client-ips", "", "Client IP Ranges and/or CIDRs from which each VU will be making requests") + flags.String("local-ips", "", "Client IP Ranges and/or CIDRs from which each VU will be making requests") flags.String("dns", types.DefaultDNSConfig().String(), "DNS resolver configuration. Possible ttl values are: 'inf' "+ "for a persistent cache, '0' to disable the cache,\nor a positive duration, e.g. '1s', '1m', etc. "+ "Milliseconds are assumed if no unit is provided.\n"+ @@ -206,12 +206,12 @@ func getOptions(flags *pflag.FlagSet) (lib.Options, error) { } } - ipRangesString, err := flags.GetString("client-ips") + localIpsString, err := flags.GetString("local-ips") if err != nil { return opts, err } - if flags.Changed("client-ips") { - err = opts.ClientIPRanges.UnmarshalText([]byte(ipRangesString)) + if flags.Changed("local-ips") { + err = opts.LocalIPs.UnmarshalText([]byte(localIpsString)) if err != nil { return opts, err } diff --git a/js/runner.go b/js/runner.go index bc404c6d3b4..87a2f1c4bcb 100644 --- a/js/runner.go +++ b/js/runner.go @@ -170,8 +170,8 @@ func (r *Runner) newVU(id int64, samplesOut chan<- stats.SampleContainer) (*VU, BlockedHostnames: r.Bundle.Options.BlockedHostnames.Trie, Hosts: r.Bundle.Options.Hosts, } - if r.Bundle.Options.ClientIPRanges.Valid { - dialer.Dialer.LocalAddr = &net.TCPAddr{IP: r.Bundle.Options.ClientIPRanges.Pool.GetIP(uint64(id - 1))} + if r.Bundle.Options.LocalIPs.Valid { + dialer.Dialer.LocalAddr = &net.TCPAddr{IP: r.Bundle.Options.LocalIPs.Pool.GetIP(uint64(id - 1))} } tlsConfig := &tls.Config{ diff --git a/lib/options.go b/lib/options.go index 85e422e0477..89deb6d2c07 100644 --- a/lib/options.go +++ b/lib/options.go @@ -390,7 +390,7 @@ type Options struct { ConsoleOutput null.String `json:"-" envconfig:"K6_CONSOLE_OUTPUT"` // Specify client IP ranges and/or CIDR from which VUs will make requests - ClientIPRanges types.NullIPPool `json:"-" envconfig:"K6_CLIENT_IPS"` + LocalIPs types.NullIPPool `json:"-" envconfig:"K6_LOCAL_IPS"` } // Returns the result of overwriting any fields with any that are set on the argument. @@ -545,8 +545,8 @@ func (o Options) Apply(opts Options) Options { if opts.ConsoleOutput.Valid { o.ConsoleOutput = opts.ConsoleOutput } - if opts.ClientIPRanges.Valid { - o.ClientIPRanges = opts.ClientIPRanges + if opts.LocalIPs.Valid { + o.LocalIPs = opts.LocalIPs } if opts.DNS.TTL.Valid { o.DNS.TTL = opts.DNS.TTL diff --git a/lib/options_test.go b/lib/options_test.go index 91f7bb8cc69..72d7aacca0c 100644 --- a/lib/options_test.go +++ b/lib/options_test.go @@ -410,8 +410,8 @@ func TestOptions(t *testing.T) { t.Run("ClientIPRanges", func(t *testing.T) { clientIPRanges, err := types.NewIPPool("129.112.232.12,123.12.0.0/32") require.NoError(t, err) - opts := Options{}.Apply(Options{ClientIPRanges: types.NullIPPool{Pool: clientIPRanges, Valid: true}}) - assert.NotNil(t, opts.ClientIPRanges) + opts := Options{}.Apply(Options{LocalIPs: types.NullIPPool{Pool: clientIPRanges, Valid: true}}) + assert.NotNil(t, opts.LocalIPs) }) } @@ -481,7 +481,7 @@ func TestOptionsEnv(t *testing.T) { "": null.String{}, "Hi!": null.StringFrom("Hi!"), }, - {"ClientIPRanges", "K6_CLIENT_IPS"}: { + {"LocalIPs", "K6_LOCAL_IPS"}: { "": types.NullIPPool{}, "192.168.220.2": types.NullIPPool{Pool: mustIPPool("192.168.220.2"), Valid: true}, "192.168.220.2/24": types.NullIPPool{Pool: mustIPPool("192.168.220.0/24"), Valid: true}, From da3f6b1222e38f068e5a3665d6f0861ee7ca48ac Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Mon, 26 Oct 2020 14:06:22 +0200 Subject: [PATCH 07/14] fixup! add ipPoolBlock for better readability --- lib/types/ipblock.go | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/types/ipblock.go b/lib/types/ipblock.go index 6d96bb93a1d..701d7051c0c 100644 --- a/lib/types/ipblock.go +++ b/lib/types/ipblock.go @@ -172,7 +172,6 @@ func (pool *IPPool) GetIP(index uint64) net.IP { func (pool *IPPool) GetIPBig(index *big.Int) net.IP { index = new(big.Int).Rem(index, pool.count) for _, b := range pool.list { - fmt.Println(index, b.startIndex) if index.Cmp(b.startIndex) >= 0 { return b.getIP(index.Sub(index, b.startIndex)) } From 645a4509d7e2d6996441e2ea206d41eeda078708 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Mon, 26 Oct 2020 14:10:06 +0200 Subject: [PATCH 08/14] Add some error cases tests --- lib/types/ipblock_test.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lib/types/ipblock_test.go b/lib/types/ipblock_test.go index 070c15a2424..20ad2b980f0 100644 --- a/lib/types/ipblock_test.go +++ b/lib/types/ipblock_test.go @@ -118,3 +118,25 @@ func TestIPPool(t *testing.T) { }) } } + +func TestIpBlockError(t *testing.T) { + testdata := map[string]string{ + "whatever": "not a valid IP", + "192.168.0.1012": "not a valid IP", + "192.168.0.10/244": "invalid CIDR", + "fd00::0/244": "invalid CIDR", + "192.168.0.101-192.168.0.102/32": "wrong IP range format", + "192.168.0.101-fd00::1": "mixed IP range format", + "fd00::1-192.168.0.101": "mixed IP range format", + "192.168.0.100-192.168.0.2": "negative IP range", + "fd00:1:1:0::0-fd00:1:0:ff::3ff": "negative IP range", + } + for name, data := range testdata { + name, data := name, data + t.Run(name, func(t *testing.T) { + _, err := getIPBlock(name) + require.Error(t, err) + require.Contains(t, err.Error(), data) + }) + } +} From 9a18622ad890e9aa2c64501b2d6ff825382ba966 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Mon, 26 Oct 2020 17:51:58 +0200 Subject: [PATCH 09/14] Better ipPoolBlock comment --- lib/types/ipblock.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/types/ipblock.go b/lib/types/ipblock.go index 701d7051c0c..0c3ab4559e1 100644 --- a/lib/types/ipblock.go +++ b/lib/types/ipblock.go @@ -34,8 +34,8 @@ type ipBlock struct { ipv6 bool } -// ipPoolBlock is almost the same as ipBlock but is put in a IPPool and knows it's start id indest -// of it's size/count +// ipPoolBlock is similar to ipBlock but instead of knowing its count/size it knows the first index +// from which it starts in an IPPool type ipPoolBlock struct { firstIP, startIndex *big.Int } From bd1c03ed876b5dba6200d34171eeff9c70a87fa9 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Mon, 26 Oct 2020 17:54:57 +0200 Subject: [PATCH 10/14] Fix reversing the ips in IPPool thanks to @divfor --- lib/types/ipblock.go | 2 +- lib/types/ipblock_test.go | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/types/ipblock.go b/lib/types/ipblock.go index 0c3ab4559e1..a016e1ed837 100644 --- a/lib/types/ipblock.go +++ b/lib/types/ipblock.go @@ -158,7 +158,7 @@ func NewIPPool(ranges string) (*IPPool, error) { // bigger than startIndex but it will be true always for the first block which is with // startIndex 0. This can also be fixed by iterating in reverse but this seems better for i := 0; i < len(pool.list)/2; i++ { - pool.list[i], pool.list[len(pool.list)/2-i] = pool.list[len(pool.list)/2-i], pool.list[i] + pool.list[i], pool.list[len(pool.list)-1-i] = pool.list[len(pool.list)-1-i], pool.list[i] } return pool, nil } diff --git a/lib/types/ipblock_test.go b/lib/types/ipblock_test.go index 20ad2b980f0..dcbddaa02af 100644 --- a/lib/types/ipblock_test.go +++ b/lib/types/ipblock_test.go @@ -105,6 +105,20 @@ func TestIPPool(t *testing.T) { 65541: net.ParseIP("192.168.0.101"), }, }, + + "192.168.0.101,192.168.0.102,192.168.0.103,192.168.0.104,192.168.0.105-192.168.0.105,fd00::2/112": { + count: new(big.Int).SetInt64(65541), + queries: map[uint64]net.IP{ + 0: net.ParseIP("192.168.0.101"), + 1: net.ParseIP("192.168.0.102"), + 2: net.ParseIP("192.168.0.103"), + 3: net.ParseIP("192.168.0.104"), + 4: net.ParseIP("192.168.0.105"), + 5: net.ParseIP("fd00::0"), + 6: net.ParseIP("fd00::1"), + 65541: net.ParseIP("192.168.0.101"), + }, + }, } for name, data := range testdata { name, data := name, data From 7e2f8d774b846e56e188a258e9f5709954ee8fb3 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Mon, 26 Oct 2020 18:13:55 +0200 Subject: [PATCH 11/14] Drop trimming client-ips everywhere --- lib/types/ipblock.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/types/ipblock.go b/lib/types/ipblock.go index a016e1ed837..0c3f18b8f45 100644 --- a/lib/types/ipblock.go +++ b/lib/types/ipblock.go @@ -62,8 +62,7 @@ func getIPBlock(s string) (*ipBlock, error) { func ipBlockFromRange(s string) (*ipBlock, error) { ss := strings.SplitN(s, "-", 2) - ip0Str, ip1Str := strings.TrimSpace(ss[0]), strings.TrimSpace(ss[1]) - ip0, ip1 := net.ParseIP(ip0Str), net.ParseIP(ip1Str) + ip0, ip1 := net.ParseIP(ss[0]), net.ParseIP(ss[1]) if ip0 == nil || ip1 == nil { return nil, errors.New("wrong IP range format: " + s) } @@ -137,7 +136,7 @@ func (b ipPoolBlock) getIP(index *big.Int) net.IP { // NewIPPool returns an IPPool slice from the provided string representation that should be comma // separated list of IPs, IP ranges(ip1-ip2) and CIDRs func NewIPPool(ranges string) (*IPPool, error) { - ss := strings.Split(strings.TrimSpace(ranges), ",") + ss := strings.Split(ranges, ",") pool := &IPPool{} pool.list = make([]ipPoolBlock, len(ss)) pool.count = new(big.Int) From 63d5f1b3dd22c206a4a547a39c9eedffef4390d3 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Tue, 27 Oct 2020 14:01:58 +0200 Subject: [PATCH 12/14] Add example for local-ips cli arg --- cmd/options.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/options.go b/cmd/options.go index 81bfbb4b47e..4dd3dbe6896 100644 --- a/cmd/options.go +++ b/cmd/options.go @@ -93,7 +93,8 @@ func optionFlagSet() *pflag.FlagSet { flags.StringSlice("tag", nil, "add a `tag` to be applied to all samples, as `[name]=[value]`") flags.String("console-output", "", "redirects the console logging to the provided output file") flags.Bool("discard-response-bodies", false, "Read but don't process or save HTTP response bodies") - flags.String("local-ips", "", "Client IP Ranges and/or CIDRs from which each VU will be making requests") + flags.String("local-ips", "", "Client IP Ranges and/or CIDRs from which each VU will be making requests, "+ + "e.g. '192.168.220.1,192.168.0.10-192.168.0.25', 'fd:1::0/120', etc.") flags.String("dns", types.DefaultDNSConfig().String(), "DNS resolver configuration. Possible ttl values are: 'inf' "+ "for a persistent cache, '0' to disable the cache,\nor a positive duration, e.g. '1s', '1m', etc. "+ "Milliseconds are assumed if no unit is provided.\n"+ From df5195526c90bd4aff6c696aaab4f5c4d46bf2dc Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Tue, 27 Oct 2020 14:06:41 +0200 Subject: [PATCH 13/14] Fix overflowing ip index on VU with id 0 --- js/runner.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/js/runner.go b/js/runner.go index 87a2f1c4bcb..dadcf4fde99 100644 --- a/js/runner.go +++ b/js/runner.go @@ -171,7 +171,11 @@ func (r *Runner) newVU(id int64, samplesOut chan<- stats.SampleContainer) (*VU, Hosts: r.Bundle.Options.Hosts, } if r.Bundle.Options.LocalIPs.Valid { - dialer.Dialer.LocalAddr = &net.TCPAddr{IP: r.Bundle.Options.LocalIPs.Pool.GetIP(uint64(id - 1))} + ipIndex := uint64(id - 1) + if id == 0 { + ipIndex = 1 + } + dialer.Dialer.LocalAddr = &net.TCPAddr{IP: r.Bundle.Options.LocalIPs.Pool.GetIP(ipIndex)} } tlsConfig := &tls.Config{ From ef7eb9d2eb80a2e89d42711abd7aa8550b8a2f34 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Tue, 27 Oct 2020 15:04:00 +0200 Subject: [PATCH 14/14] Update js/runner.go Co-authored-by: na-- --- js/runner.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/js/runner.go b/js/runner.go index dadcf4fde99..78629479ba7 100644 --- a/js/runner.go +++ b/js/runner.go @@ -171,9 +171,9 @@ func (r *Runner) newVU(id int64, samplesOut chan<- stats.SampleContainer) (*VU, Hosts: r.Bundle.Options.Hosts, } if r.Bundle.Options.LocalIPs.Valid { - ipIndex := uint64(id - 1) - if id == 0 { - ipIndex = 1 + var ipIndex uint64 + if id > 0 { + ipIndex = uint64(id - 1) } dialer.Dialer.LocalAddr = &net.TCPAddr{IP: r.Bundle.Options.LocalIPs.Pool.GetIP(ipIndex)} }