:[WIP] Add dual-stack support

Signed-off-by: Manuel Buil <mbuil@suse.com>

go.mod

@@ -73,7 +73,6 @@ replace (
 
 require (
 	github.com/Microsoft/hcsshim v0.8.20
-	github.com/bronze1man/goStrongswanVici v0.0.0-20190828090544-27d02f80ba40 // indirect
 	github.com/containerd/cgroups v1.0.1
 	github.com/containerd/containerd v1.5.5
 	github.com/containerd/fuse-overlayfs-snapshotter v1.0.3
@@ -82,7 +81,7 @@ require (
 	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f
 	github.com/docker/docker v20.10.7+incompatible
 	github.com/erikdubbelboer/gspt v0.0.0-20190125194910-e68493906b83
-	github.com/flannel-io/flannel v0.14.0
+	github.com/flannel-io/flannel v0.14.1-0.20210827074410-fca1560c91cc
 	github.com/go-bindata/go-bindata v3.1.2+incompatible
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/golangplus/testing v1.0.0 // indirect

go.sum

@@ -119,9 +119,8 @@ github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
-github.com/bronze1man/goStrongswanVici v0.0.0-20171013065002-4d72634a2f11/go.mod h1:c+n7HXa5FxzR8GDsmu773UtbtrmKvMVerLVQeEbnzAE=
-github.com/bronze1man/goStrongswanVici v0.0.0-20190828090544-27d02f80ba40 h1:udTfdeYqe866Z5mxTaEm5irSJK2vupyxwBOHAYEVtJo=
-github.com/bronze1man/goStrongswanVici v0.0.0-20190828090544-27d02f80ba40/go.mod h1:fWUtBEPt2yjrr3WFhOqvajM8JSEU8bEeBcoeSCsKRpc=
+github.com/bronze1man/goStrongswanVici v0.0.0-20201105010758-936f38b697fd h1:qn6a8rGrW+7p4ghypmYHZUKewXURuUDYxKqZxEoFjPc=
+github.com/bronze1man/goStrongswanVici v0.0.0-20201105010758-936f38b697fd/go.mod h1:fWUtBEPt2yjrr3WFhOqvajM8JSEU8bEeBcoeSCsKRpc=
 github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
 github.com/canonical/go-dqlite v1.5.1 h1:1YjtIrFsC1A3XlgsX38ARAiKhvkZS63PqsEd8z3T4yU=
 github.com/canonical/go-dqlite v1.5.1/go.mod h1:wp00vfMvPYgNCyxcPdHB5XExmDoCGoPUGymloAQT17Y=
@@ -284,8 +283,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ=
 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
-github.com/flannel-io/flannel v0.14.0 h1:7RmZN6G0L/mJwdzsLrAjfQKtKokfD1NcncPEfSqr+ac=
-github.com/flannel-io/flannel v0.14.0/go.mod h1:qZhrC3nxQudgshBtTb5rBqFxeYtQGRa4AQGwKi4u4Ds=
+github.com/flannel-io/flannel v0.14.1-0.20210827074410-fca1560c91cc h1:t/L/tIYcAayH7XICVYtAscY/PXaJDKXsKheAMCtiKS0=
+github.com/flannel-io/flannel v0.14.1-0.20210827074410-fca1560c91cc/go.mod h1:fIcQpjXVBEE22oxqfZN0cXN0ZInsMDqTF5YoeGo6DgY=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=

install.sh

@@ -648,6 +648,7 @@ ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore;
 done
 ip link delete cni0
 ip link delete flannel.1
+ip link delete flannel-v6.1
 rm -rf /var/lib/cni/
 iptables-save | grep -v KUBE- | grep -v CNI- | iptables-restore
 EOF

pkg/agent/flannel/flannel.go

@@ -39,13 +39,13 @@ const (
 	subnetFile = "/run/flannel/subnet.env"
 )
 
-func flannel(ctx context.Context, flannelIface *net.Interface, flannelConf, kubeConfigFile string) error {
-	extIface, err := LookupExtInterface(flannelIface)
+func flannel(ctx context.Context, flannelIface *net.Interface, flannelConf, kubeConfigFile string, netMode int) error {
+	extIface, err := LookupExtInterface(flannelIface, netMode)
 	if err != nil {
 		return err
 	}
 
-	sm, err := kube.NewSubnetManager(ctx, "", kubeConfigFile, "flannel.alpha.coreos.com", flannelConf)
+	sm, err := kube.NewSubnetManager(ctx, "", kubeConfigFile, "flannel.alpha.coreos.com", flannelConf, false)
 	if err != nil {
 		return err
 	}
@@ -71,7 +71,7 @@ func flannel(ctx context.Context, flannelIface *net.Interface, flannelConf, kube
 	go network.SetupAndEnsureIPTables(network.MasqRules(config.Network, bn.Lease()), 60)
 	go network.SetupAndEnsureIPTables(network.ForwardRules(config.Network.String()), 50)
 
-	if err := WriteSubnetFile(subnetFile, config.Network, true, bn); err != nil {
+	if err := WriteSubnetFile(subnetFile, config.Network, config.IPv6Network, true, bn); err != nil {
 		// Continue, even though it failed.
 		log.Warningf("Failed to write subnet file: %s", err)
 	} else {
@@ -84,8 +84,9 @@ func flannel(ctx context.Context, flannelIface *net.Interface, flannelConf, kube
 	return nil
 }
 
-func LookupExtInterface(iface *net.Interface) (*backend.ExternalInterface, error) {
+func LookupExtInterface(iface *net.Interface, netMode int) (*backend.ExternalInterface, error) {
 	var ifaceAddr net.IP
+	var ifacev6Addr net.IP
 	var err error
 
 	if iface == nil {
@@ -94,28 +95,42 @@ func LookupExtInterface(iface *net.Interface) (*backend.ExternalInterface, error
 			return nil, fmt.Errorf("failed to get default interface: %s", err)
 		}
 	} else {
-		log.Info("Determining IP address of specified interface: ", iface.Name)
+		log.Infof("Using interface with name %s and address %s", iface.Name, ifaceAddr)
 	}
 
 	ifaceAddr, err = ip.GetInterfaceIP4Addr(iface)
 	if err != nil {
 		return nil, fmt.Errorf("failed to find IPv4 address for interface %s", iface.Name)
 	}
 
-	log.Infof("Using interface with name %s and address %s", iface.Name, ifaceAddr)
+	if netMode == (ipv4 + ipv6) {
+		ifacev6Addr, err = ip.GetInterfaceIP6Addr(iface)
+		if err != nil {
+			return nil, fmt.Errorf("failed to find IPv6 address for interface %s", iface.Name)
+		}
+
+		_, err := ip.GetDefaultV6GatewayInterface()
+
+		if err != nil {
+			return nil, fmt.Errorf("No ipv6 default route. Please add one")
+		}
 
+		log.Infof("Using ipv6 address %s", ifacev6Addr)
+	}
 	if iface.MTU == 0 {
 		return nil, fmt.Errorf("failed to determine MTU for %s interface", ifaceAddr)
 	}
 
 	return &backend.ExternalInterface{
-		Iface:     iface,
-		IfaceAddr: ifaceAddr,
-		ExtAddr:   ifaceAddr,
+		Iface:       iface,
+		IfaceAddr:   ifaceAddr,
+		IfaceV6Addr: ifacev6Addr,
+		ExtAddr:     ifaceAddr,
+		ExtV6Addr:   ifacev6Addr,
 	}, nil
 }
 
-func WriteSubnetFile(path string, nw ip.IP4Net, ipMasq bool, bn backend.Network) error {
+func WriteSubnetFile(path string, nw ip.IP4Net, nwv6 ip.IP6Net, ipMasq bool, bn backend.Network) error {
 	dir, name := filepath.Split(path)
 	os.MkdirAll(dir, 0755)
 
@@ -132,6 +147,14 @@ func WriteSubnetFile(path string, nw ip.IP4Net, ipMasq bool, bn backend.Network)
 
 	fmt.Fprintf(f, "FLANNEL_NETWORK=%s\n", nw)
 	fmt.Fprintf(f, "FLANNEL_SUBNET=%s\n", sn)
+
+	if nwv6.String() != emptyIPv6Network {
+		snv6 := bn.Lease().IPv6Subnet
+		snv6.IncrementIP()
+		fmt.Fprintf(f, "FLANNEL_IPV6_NETWORK=%s\n", nwv6)
+		fmt.Fprintf(f, "FLANNEL_IPV6_SUBNET=%s\n", snv6)
+	}
+
 	fmt.Fprintf(f, "FLANNEL_MTU=%d\n", bn.MTU())
 	_, err = fmt.Fprintf(f, "FLANNEL_IPMASQ=%v\n", ipMasq)
 	f.Close()

pkg/agent/flannel/setup.go

@@ -2,7 +2,9 @@ package flannel
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"net"
 	"os"
 	"path/filepath"
 	"strings"
@@ -14,6 +16,7 @@ import (
 	"github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	v1 "k8s.io/client-go/kubernetes/typed/core/v1"
+	utilsnet "k8s.io/utils/net"
 )
 
 const (
@@ -41,6 +44,8 @@ const (
 
 	flannelConf = `{
 	"Network": "%CIDR%",
+	"EnableIPv6": %DUALSTACK%,
+	"IPv6Network": "%CIDR_IPV6%",
 	"Backend": %backend%
 }
 `
@@ -67,6 +72,11 @@ const (
 	"SubnetAddCommand": "read PUBLICKEY; wg set flannel.1 peer $PUBLICKEY endpoint $PUBLIC_IP:51820 allowed-ips $SUBNET persistent-keepalive 25",
 	"SubnetRemoveCommand": "read PUBLICKEY; wg set flannel.1 peer $PUBLICKEY remove"
 }`
+
+	emptyIPv6Network = "::/0"
+
+	ipv4 = iota + 1
+	ipv6
 )
 
 func Prepare(ctx context.Context, nodeConfig *config.Node) error {
@@ -94,8 +104,13 @@ func Run(ctx context.Context, nodeConfig *config.Node, nodes v1.NodeInterface) e
 	}
 	logrus.Info("Node CIDR assigned for: " + nodeName)
 
+	netMode, err := findNetMode(nodeConfig.AgentConfig.ClusterCIDRs)
+	if err != nil {
+		logrus.Fatalf("Error checking netMode")
+		return err
+	}
 	go func() {
-		err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConf, nodeConfig.AgentConfig.KubeConfigKubelet)
+		err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConf, nodeConfig.AgentConfig.KubeConfigKubelet, netMode)
 		logrus.Fatalf("flannel exited: %v", err)
 	}()
 
@@ -139,6 +154,24 @@ func createFlannelConf(nodeConfig *config.Node) error {
 	}
 	confJSON = strings.ReplaceAll(confJSON, "%backend%", backendConf)
 
+	netMode, err := findNetMode(nodeConfig.AgentConfig.ClusterCIDRs)
+	if err != nil {
+		logrus.Fatalf("Error checking netMode")
+		return err
+	}
+
+	if netMode == (ipv4 + ipv6) {
+		confJSON = strings.ReplaceAll(confJSON, "%DUALSTACK%", "true")
+		for _, cidr := range nodeConfig.AgentConfig.ClusterCIDRs {
+			if utilsnet.IsIPv6(cidr.IP) {
+				// Assuming there is going to be only one ipv6 range
+				confJSON = strings.ReplaceAll(confJSON, "%CIDR_IPV6%", cidr.String())
+			}
+		}
+	} else {
+		confJSON = strings.ReplaceAll(confJSON, "%DUALSTACK%", "false")
+		confJSON = strings.ReplaceAll(confJSON, "%CIDR_IPV6%", emptyIPv6Network)
+	}
 	return util.WriteFile(nodeConfig.FlannelConf, confJSON)
 }
 
@@ -169,3 +202,24 @@ func setupStrongSwan(nodeConfig *config.Node) error {
 	// make new strongswan link
 	return os.Symlink(dataDir, nodeConfig.AgentConfig.StrongSwanDir)
 }
+
+// fundNetMode returns the mode (ipv4, ipv6 or dual-stack) in which flannel is operating
+func findNetMode(cidrs []*net.IPNet) (int, error) {
+	dualStack, err := utilsnet.IsDualStackCIDRs(cidrs)
+	if err != nil {
+		return 0, err
+	}
+	if dualStack {
+		return ipv4 + ipv6, nil
+	}
+
+	for _, cidr := range cidrs {
+		if utilsnet.IsIPv4CIDR(cidr) {
+			return ipv4, nil
+		}
+		if utilsnet.IsIPv6CIDR(cidr) {
+			return ipv6, nil
+		}
+	}
+	return 0, errors.New("Failed checking netMode")
+}

scripts/build

@@ -95,7 +95,7 @@ if [ ! -x ${INSTALLBIN}/cni ]; then
     TMPDIR=$(mktemp -d)
     trap cleanup EXIT
     WORKDIR=$TMPDIR/src/github.com/containernetworking/plugins
-    git clone -b $VERSION_CNIPLUGINS https://github.com/rancher/plugins.git $WORKDIR
+    git clone -b $VERSION_CNIPLUGINS https://github.com/manuelbuil/plugins.git $WORKDIR
     cd $WORKDIR
     GOPATH=$TMPDIR CGO_ENABLED=0 "${GO}" build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC" -o $INSTALLBIN/cni
 )