libnet/ipam: Lazily sub-divide pools into subnets

A new Subnetter structure is added to lazily sub-divide an address pool
into subnets. This fixes #40275.

Prior to this change, the list of NetworkToSplit was eagerly split into
smaller subnets when ipamutils package was loaded, when
ConfigGlobalScopeDefaultNetworks was called or when the function
SetDefaultIPAddressPool from the default IPAM driver was called. In the
latter case, if the list of NetworkToSplit contained an IPv6 prefix,
eagerly enumerating all subnets could eat all the available memory. For
instance, fd00::/8 split into /96 would take ~5*10^27 bytes.

Although this change trades memory consumption for computation cost, the
Subnetter is used by libnetwork/ipam package in such a way that it
only have to compute the address of the next subnet. When
the Subnetter reach the end of NetworkToSplit, it's resetted by
libnetwork/ipam only if there were some subnets released beforehand. In
such case, ipam package might iterate over all the subnets before
finding one available.

Also, the Subnetter leverages the newly introduced ipbits package, which
handles IPv6 addresses correctly. Before this commit, a bitwise shift
was overflowing uint64 and thus only a single subnet could be enumerated
from an IPv6 prefix. This fixes #42801.

Signed-off-by: Albin Kerouanton <albinker@gmail.com>

daemon/config/config_test.go

@@ -165,7 +165,8 @@ func TestDaemonConfigurationMergeDefaultAddressPools(t *testing.T) {
 
 		config, err := MergeDaemonConfigurations(&conf, flags, emptyConfigFile)
 		assert.NilError(t, err)
-		assert.DeepEqual(t, config.DefaultAddressPools.Value(), expected)
+		assert.DeepEqual(t, config.DefaultAddressPools.Value(), expected,
+			cmpopts.IgnoreUnexported(ipamutils.NetworkToSplit{}))
 	})
 
 	t.Run("config file", func(t *testing.T) {
@@ -175,7 +176,8 @@ func TestDaemonConfigurationMergeDefaultAddressPools(t *testing.T) {
 
 		config, err := MergeDaemonConfigurations(&conf, flags, configFile)
 		assert.NilError(t, err)
-		assert.DeepEqual(t, config.DefaultAddressPools.Value(), expected)
+		assert.DeepEqual(t, config.DefaultAddressPools.Value(), expected,
+			cmpopts.IgnoreUnexported(ipamutils.NetworkToSplit{}))
 	})
 
 	t.Run("with conflicting options", func(t *testing.T) {

libnetwork/drivers/bridge/bridge_test.go

@@ -168,9 +168,21 @@ func compareBindings(a, b []types.PortBinding) bool {
 	return true
 }
 
+func retrieveAllSubnets(s ipamutils.Subnetter) []*net.IPNet {
+	nets := make([]*net.IPNet, 0)
+	for nw, err := s.NextSubnet(); err == nil; nw, err = s.NextSubnet() {
+		nets = append(nets, &net.IPNet{
+			IP:   nw.Addr().AsSlice(),
+			Mask: net.CIDRMask(nw.Bits(), nw.Addr().BitLen()),
+		})
+	}
+
+	return nets
+}
+
 func getIPv4Data(t *testing.T, iface string) []driverapi.IPAMData {
 	ipd := driverapi.IPAMData{AddressSpace: "full"}
-	nw, err := netutils.FindAvailableNetwork(ipamutils.GetLocalScopeDefaultNetworks())
+	nw, err := netutils.FindAvailableNetwork(retrieveAllSubnets(ipamutils.GetDefaultLocalScopeSubnetter()))
 	if err != nil {
 		t.Fatal(err)
 	}

libnetwork/ipam/allocator.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/docker/docker/libnetwork/bitmap"
 	"github.com/docker/docker/libnetwork/ipamapi"
+	"github.com/docker/docker/libnetwork/ipamutils"
 	"github.com/docker/docker/libnetwork/ipbits"
 	"github.com/docker/docker/libnetwork/types"
 	"github.com/sirupsen/logrus"
@@ -25,7 +26,7 @@ type Allocator struct {
 }
 
 // NewAllocator returns an instance of libnetwork ipam
-func NewAllocator(lcAs, glAs []*net.IPNet) (*Allocator, error) {
+func NewAllocator(lcAs, glAs ipamutils.Subnetter) (*Allocator, error) {
 	var (
 		a   Allocator
 		err error
@@ -41,18 +42,14 @@ func NewAllocator(lcAs, glAs []*net.IPNet) (*Allocator, error) {
 	return &a, nil
 }
 
-func newAddrSpace(predefined []*net.IPNet) (*addrSpace, error) {
-	pdf := make([]netip.Prefix, len(predefined))
-	for i, n := range predefined {
-		var ok bool
-		pdf[i], ok = toPrefix(n)
-		if !ok {
-			return nil, fmt.Errorf("network at index %d (%v) is not in canonical form", i, n)
-		}
-	}
+func newAddrSpace(predefined ipamutils.Subnetter) (*addrSpace, error) {
 	return &addrSpace{
-		subnets:    map[netip.Prefix]*PoolData{},
-		predefined: pdf,
+		subnets: map[netip.Prefix]*PoolData{},
+		predefined: map[ipamutils.IPVersion]*ipamutils.Subnetter{
+			ipamutils.IPv4: predefined.V4(),
+			ipamutils.IPv6: predefined.V6(),
+		},
+		reset: map[ipamutils.IPVersion]bool{},
 	}, nil
 }
 
@@ -169,57 +166,42 @@ func newPoolData(pool netip.Prefix) *PoolData {
 	return &PoolData{addrs: h, children: map[netip.Prefix]struct{}{}}
 }
 
-// getPredefineds returns the predefined subnets for the address space.
-//
-// It should not be called concurrently with any other method on the addrSpace.
-func (aSpace *addrSpace) getPredefineds() []netip.Prefix {
-	i := aSpace.predefinedStartIndex
-	// defensive in case the list changed since last update
-	if i >= len(aSpace.predefined) {
-		i = 0
-	}
-	return append(aSpace.predefined[i:], aSpace.predefined[:i]...)
-}
-
-// updatePredefinedStartIndex rotates the predefined subnet list by amt.
-//
-// It should not be called concurrently with any other method on the addrSpace.
-func (aSpace *addrSpace) updatePredefinedStartIndex(amt int) {
-	i := aSpace.predefinedStartIndex + amt
-	if i < 0 || i >= len(aSpace.predefined) {
-		i = 0
+func (aSpace *addrSpace) allocatePredefinedPool(ipV6 bool) (netip.Prefix, error) {
+	v := ipamutils.IPv4
+	if ipV6 {
+		v = ipamutils.IPv6
 	}
-	aSpace.predefinedStartIndex = i
-}
 
-func (aSpace *addrSpace) allocatePredefinedPool(ipV6 bool) (netip.Prefix, error) {
 	aSpace.Lock()
 	defer aSpace.Unlock()
 
-	for i, nw := range aSpace.getPredefineds() {
-		if ipV6 != nw.Addr().Is6() {
-			continue
-		}
-		// Checks whether pool has already been allocated
+	s := aSpace.predefined[v]
+	if s.EndReached() && aSpace.reset[v] {
+		s.Reset()
+		aSpace.reset[v] = false
+	}
+
+	for nw, err := s.NextSubnet(); err == nil; nw, err = s.NextSubnet() {
+		// Checks whether sub-pool has already been allocated
 		if _, ok := aSpace.subnets[nw]; ok {
 			continue
 		}
 		// Shouldn't be necessary, but check prevents IP collisions should
 		// predefined pools overlap for any reason.
 		if !aSpace.contains(nw) {
-			aSpace.updatePredefinedStartIndex(i + 1)
 			err := aSpace.allocateSubnetL(nw, netip.Prefix{})
 			if err != nil {
 				return netip.Prefix{}, err
 			}
 			return nw, nil
 		}
-	}
 
-	v := 4
-	if ipV6 {
-		v = 6
+		if s.EndReached() && aSpace.reset[v] {
+			s.Reset()
+			aSpace.reset[v] = false
+		}
 	}
+
 	return netip.Prefix{}, types.NotFoundErrorf("could not find an available, non-overlapping IPv%d address pool among the defaults to assign to the network", v)
 }
 

libnetwork/ipam/allocator_test.go

@@ -54,7 +54,7 @@ func TestKeyString(t *testing.T) {
 }
 
 func TestAddSubnets(t *testing.T) {
-	a, err := NewAllocator(ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
+	a, err := NewAllocator(ipamutils.GetDefaultLocalScopeSubnetter(), ipamutils.GetDefaultGlobalScopeSubnetter())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -116,7 +116,7 @@ func TestAddSubnets(t *testing.T) {
 // TestDoublePoolRelease tests that releasing a pool which has already
 // been released raises an error.
 func TestDoublePoolRelease(t *testing.T) {
-	a, err := NewAllocator(ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
+	a, err := NewAllocator(ipamutils.GetDefaultLocalScopeSubnetter(), ipamutils.GetDefaultGlobalScopeSubnetter())
 	assert.NilError(t, err)
 
 	pid0, _, _, err := a.RequestPool(localAddressSpace, "10.0.0.0/8", "", nil, false)
@@ -130,7 +130,7 @@ func TestDoublePoolRelease(t *testing.T) {
 }
 
 func TestAddReleasePoolID(t *testing.T) {
-	a, err := NewAllocator(ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
+	a, err := NewAllocator(ipamutils.GetDefaultLocalScopeSubnetter(), ipamutils.GetDefaultGlobalScopeSubnetter())
 	assert.NilError(t, err)
 
 	var k0, k1 PoolID
@@ -257,7 +257,7 @@ func TestAddReleasePoolID(t *testing.T) {
 }
 
 func TestPredefinedPool(t *testing.T) {
-	a, err := NewAllocator(ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
+	a, err := NewAllocator(ipamutils.GetDefaultLocalScopeSubnetter(), ipamutils.GetDefaultGlobalScopeSubnetter())
 	assert.NilError(t, err)
 
 	pid, nw, _, err := a.RequestPool(localAddressSpace, "", "", nil, false)
@@ -284,7 +284,7 @@ func TestPredefinedPool(t *testing.T) {
 }
 
 func TestRemoveSubnet(t *testing.T) {
-	a, err := NewAllocator(ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
+	a, err := NewAllocator(ipamutils.GetDefaultLocalScopeSubnetter(), ipamutils.GetDefaultGlobalScopeSubnetter())
 	assert.NilError(t, err)
 
 	input := []struct {
@@ -318,7 +318,7 @@ func TestRemoveSubnet(t *testing.T) {
 }
 
 func TestGetSameAddress(t *testing.T) {
-	a, err := NewAllocator(ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
+	a, err := NewAllocator(ipamutils.GetDefaultLocalScopeSubnetter(), ipamutils.GetDefaultGlobalScopeSubnetter())
 	assert.NilError(t, err)
 
 	pid, _, _, err := a.RequestPool(localAddressSpace, "192.168.100.0/24", "", nil, false)
@@ -339,7 +339,7 @@ func TestGetSameAddress(t *testing.T) {
 }
 
 func TestPoolAllocationReuse(t *testing.T) {
-	a, err := NewAllocator(ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
+	a, err := NewAllocator(ipamutils.GetDefaultLocalScopeSubnetter(), ipamutils.GetDefaultGlobalScopeSubnetter())
 	assert.NilError(t, err)
 
 	// First get all pools until they are exhausted to
@@ -376,7 +376,7 @@ func TestPoolAllocationReuse(t *testing.T) {
 }
 
 func TestGetAddressSubPoolEqualPool(t *testing.T) {
-	a, err := NewAllocator(ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
+	a, err := NewAllocator(ipamutils.GetDefaultLocalScopeSubnetter(), ipamutils.GetDefaultGlobalScopeSubnetter())
 	assert.NilError(t, err)
 
 	// Requesting a subpool of same size of the master pool should not cause any problem on ip allocation
@@ -392,7 +392,7 @@ func TestGetAddressSubPoolEqualPool(t *testing.T) {
 }
 
 func TestRequestReleaseAddressFromSubPool(t *testing.T) {
-	a, err := NewAllocator(ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
+	a, err := NewAllocator(ipamutils.GetDefaultLocalScopeSubnetter(), ipamutils.GetDefaultGlobalScopeSubnetter())
 	assert.NilError(t, err)
 
 	poolID, _, _, err := a.RequestPool(localAddressSpace, "172.28.0.0/16", "172.28.30.0/24", nil, false)
@@ -515,7 +515,7 @@ func TestRequestReleaseAddressFromSubPool(t *testing.T) {
 func TestSerializeRequestReleaseAddressFromSubPool(t *testing.T) {
 	opts := map[string]string{
 		ipamapi.AllocSerialPrefix: "true"}
-	a, err := NewAllocator(ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
+	a, err := NewAllocator(ipamutils.GetDefaultLocalScopeSubnetter(), ipamutils.GetDefaultGlobalScopeSubnetter())
 	assert.NilError(t, err)
 
 	poolID, _, _, err := a.RequestPool(localAddressSpace, "172.28.0.0/16", "172.28.30.0/24", nil, false)
@@ -654,7 +654,7 @@ func TestRequestSyntaxCheck(t *testing.T) {
 		subPool = "192.168.0.0/24"
 	)
 
-	a, err := NewAllocator(ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
+	a, err := NewAllocator(ipamutils.GetDefaultLocalScopeSubnetter(), ipamutils.GetDefaultGlobalScopeSubnetter())
 	assert.NilError(t, err)
 
 	_, _, _, err = a.RequestPool("", pool, "", nil, false)
@@ -800,7 +800,7 @@ func TestOverlappingRequests(t *testing.T) {
 	}
 
 	for _, tc := range input {
-		a, err := NewAllocator(ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
+		a, err := NewAllocator(ipamutils.GetDefaultLocalScopeSubnetter(), ipamutils.GetDefaultGlobalScopeSubnetter())
 		assert.NilError(t, err)
 
 		// Set up some existing allocations.  This should always succeed.
@@ -837,7 +837,7 @@ func TestUnusualSubnets(t *testing.T) {
 		{"192.168.0.3"},
 	}
 
-	allocator, err := NewAllocator(ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
+	allocator, err := NewAllocator(ipamutils.GetDefaultLocalScopeSubnetter(), ipamutils.GetDefaultGlobalScopeSubnetter())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -885,7 +885,7 @@ func TestRelease(t *testing.T) {
 		subnet = "192.168.0.0/23"
 	)
 
-	a, err := NewAllocator(ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
+	a, err := NewAllocator(ipamutils.GetDefaultLocalScopeSubnetter(), ipamutils.GetDefaultGlobalScopeSubnetter())
 	assert.NilError(t, err)
 
 	pid, _, _, err := a.RequestPool(localAddressSpace, subnet, "", nil, false)
@@ -985,7 +985,7 @@ func assertNRequests(t *testing.T, subnet string, numReq int, lastExpectedIP str
 	)
 
 	lastIP := net.ParseIP(lastExpectedIP)
-	a, err := NewAllocator(ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
+	a, err := NewAllocator(ipamutils.GetDefaultLocalScopeSubnetter(), ipamutils.GetDefaultGlobalScopeSubnetter())
 	assert.NilError(t, err)
 
 	pid, _, _, err := a.RequestPool(localAddressSpace, subnet, "", nil, false)
@@ -1027,7 +1027,7 @@ func BenchmarkRequest(b *testing.B) {
 	for _, subnet := range subnets {
 		name := fmt.Sprintf("%vSubnet", subnet)
 		b.Run(name, func(b *testing.B) {
-			a, _ := NewAllocator(ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
+			a, _ := NewAllocator(ipamutils.GetDefaultLocalScopeSubnetter(), ipamutils.GetDefaultGlobalScopeSubnetter())
 			benchmarkRequest(b, a, subnet)
 		})
 	}
@@ -1041,7 +1041,7 @@ func TestAllocateRandomDeallocate(t *testing.T) {
 }
 
 func testAllocateRandomDeallocate(t *testing.T, pool, subPool string, num int, store bool) {
-	a, err := NewAllocator(ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
+	a, err := NewAllocator(ipamutils.GetDefaultLocalScopeSubnetter(), ipamutils.GetDefaultGlobalScopeSubnetter())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1135,7 +1135,7 @@ func runParallelTests(t *testing.T, instance int) {
 	// The first instance creates the allocator, gives the start
 	// and finally checks the pools each instance was assigned
 	if instance == first {
-		allocator, err = NewAllocator(ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
+		allocator, err = NewAllocator(ipamutils.GetDefaultLocalScopeSubnetter(), ipamutils.GetDefaultGlobalScopeSubnetter())
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -1167,7 +1167,7 @@ func runParallelTests(t *testing.T, instance int) {
 }
 
 func TestRequestReleaseAddressDuplicate(t *testing.T) {
-	a, err := NewAllocator(ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
+	a, err := NewAllocator(ipamutils.GetDefaultLocalScopeSubnetter(), ipamutils.GetDefaultGlobalScopeSubnetter())
 	if err != nil {
 		t.Fatal(err)
 	}

libnetwork/ipam/parallel_test.go

@@ -37,7 +37,7 @@ type testContext struct {
 }
 
 func newTestContext(t *testing.T, mask int, options map[string]string) *testContext {
-	a, err := NewAllocator(ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
+	a, err := NewAllocator(ipamutils.GetDefaultLocalScopeSubnetter(), ipamutils.GetDefaultGlobalScopeSubnetter())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -77,7 +77,7 @@ func (o *op) String() string {
 }
 
 func TestRequestPoolParallel(t *testing.T) {
-	a, err := NewAllocator(ipamutils.GetLocalScopeDefaultNetworks(), ipamutils.GetGlobalScopeDefaultNetworks())
+	a, err := NewAllocator(ipamutils.GetDefaultLocalScopeSubnetter(), ipamutils.GetDefaultGlobalScopeSubnetter())
 	if err != nil {
 		t.Fatal(err)
 	}

libnetwork/ipam/structures.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/docker/docker/libnetwork/bitmap"
 	"github.com/docker/docker/libnetwork/ipamapi"
+	"github.com/docker/docker/libnetwork/ipamutils"
 	"github.com/docker/docker/libnetwork/types"
 )
 
@@ -37,8 +38,11 @@ type addrSpace struct {
 	subnets map[netip.Prefix]*PoolData
 
 	// Predefined pool for the address space
-	predefined           []netip.Prefix
-	predefinedStartIndex int
+	predefined map[ipamutils.IPVersion]*ipamutils.Subnetter
+
+	// reset indicates whether the predefined subnetters can be reset when all subnets
+	// have been enumerated but some have been released.
+	reset map[ipamutils.IPVersion]bool
 
 	sync.Mutex
 }
@@ -156,6 +160,8 @@ func (aSpace *addrSpace) releaseSubnet(nw, sub netip.Prefix) error {
 		p.autoRelease = true
 	}
 
+	aSpace.reset[ipamutils.IPVerFromPrefix(nw)] = true
+
 	if len(p.children) == 0 && p.autoRelease {
 		delete(aSpace.subnets, nw)
 	}