Skip to content

Commit

Permalink
Implement multiple public IP resolvers
Browse files Browse the repository at this point in the history
The gateway nodes now can be labeled as:

gateway.submariner.io/public-ip=<resolver>[,resolver..]
where resolvers take the form of <method>:<parameter>

the implemented methods are:

- api
- lb
- ipv4
- dns

this commit also adds additional fallback API public-ip resolvers

In addition to https://api.ipify.org
- https://api.my-ip.io/ip
- https://ip4.seeip.org

Signed-off-by: Miguel Angel Ajo <majopela@redhat.com>
  • Loading branch information
mangelajo authored and tpantelis committed May 21, 2021
1 parent 51c4ac8 commit ca6d02e
Show file tree
Hide file tree
Showing 9 changed files with 363 additions and 22 deletions.
1 change: 0 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ require (
github.com/onsi/gomega v1.12.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.10.0
github.com/rdegges/go-ipify v0.0.0-20150526035502-2d94a6a86c40
github.com/submariner-io/admiral v0.9.0-rc0.0.20210506031438-f6fdcbce358a
github.com/submariner-io/shipyard v0.9.1-0.20210506024409-3beff067454a
github.com/vishvananda/netlink v1.1.0
Expand Down
2 changes: 0 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -746,8 +746,6 @@ github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/rdegges/go-ipify v0.0.0-20150526035502-2d94a6a86c40 h1:31Y7UZ1yTYBU4E79CE52I/1IRi3TqiuwquXGNtZDXWs=
github.com/rdegges/go-ipify v0.0.0-20150526035502-2d94a6a86c40/go.mod h1:j4c6zEU0eMG1oiZPUy+zD4ykX0NIpjZAEOEAviTWC18=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af h1:gu+uRPtBe88sKxUCEXRoeCvVG90TJmwhiqRpvdhQFng=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4 h1:BN/Nyn2nWMoqGRA7G7paDNDqTXE30mXGqzzybrfo05w=
Expand Down
8 changes: 3 additions & 5 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,7 @@ import (
"github.com/submariner-io/submariner/pkg/controllers/tunnel"
"github.com/submariner-io/submariner/pkg/endpoint"
"github.com/submariner-io/submariner/pkg/natdiscovery"
"github.com/submariner-io/submariner/pkg/node"
"github.com/submariner-io/submariner/pkg/types"
"github.com/submariner-io/submariner/pkg/util"
)

var (
Expand Down Expand Up @@ -112,9 +110,9 @@ func main() {
klog.Fatalf("Error creating submariner clientset: %s", err.Error())
}

localNode, err := node.GetLocalNode(cfg)
k8sClient, err := kubernetes.NewForConfig(cfg)
if err != nil {
klog.Fatalf("Error getting information on the local node: %s", err.Error())
klog.Fatalf("Error creating Kubernetes clientset: %s", err.Error())
}

klog.Info("Creating the cable engine")
Expand All @@ -127,7 +125,7 @@ func main() {

submSpec.CableDriver = strings.ToLower(submSpec.CableDriver)

localEndpoint, err := endpoint.GetLocal(submSpec, util.GetLocalIP(), localNode)
localEndpoint, err := endpoint.GetLocal(submSpec, k8sClient)

if err != nil {
klog.Fatalf("Error creating local endpoint object from %#v: %v", submSpec, err)
Expand Down
10 changes: 10 additions & 0 deletions pkg/apis/submariner.io/v1/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,12 +87,21 @@ const (
GatewayConfigLabelPrefix = "gateway.submariner.io/"
UDPPortConfig = "udp-port"
NATTDiscoveryPortConfig = "natt-discovery-port"
PublicIP = "public-ip"
)

const (
DefaultNATTDiscoveryPort = "4490"
)

// Valid PublicIP resolvers.
const (
IPv4 = "ipv4" // ipv4:1.2.3.4
LoadBalancer = "lb" // lb:external-gw-lb
API = "api" // api:api.ipify.org
DNS = "dns" // dns:mygateway.dns.name.com
)

// BackendConfig entries which aren't configured via labels, but exposed from the endpoints
const (
PreferredServerConfig = "preferred-server"
Expand All @@ -102,6 +111,7 @@ const (
var ValidGatewayNodeConfig = []string{
UDPPortConfig,
NATTDiscoveryPortConfig,
PublicIP,
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
Expand Down
15 changes: 12 additions & 3 deletions pkg/endpoint/local_endpoint.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,17 +26,26 @@ import (
"time"

"github.com/pkg/errors"
"github.com/rdegges/go-ipify"
"github.com/submariner-io/admiral/pkg/log"
"github.com/submariner-io/admiral/pkg/stringset"
"github.com/submariner-io/submariner/pkg/node"
"github.com/submariner-io/submariner/pkg/util"
v1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/klog"

submv1 "github.com/submariner-io/submariner/pkg/apis/submariner.io/v1"
"github.com/submariner-io/submariner/pkg/types"
)

func GetLocal(submSpec types.SubmarinerSpecification, privateIP string, gwNode *v1.Node) (types.SubmarinerEndpoint, error) {
func GetLocal(submSpec types.SubmarinerSpecification, k8sClient kubernetes.Interface) (types.SubmarinerEndpoint, error) {
privateIP := util.GetLocalIP()

gwNode, err := node.GetLocalNode(k8sClient)
if err != nil {
klog.Fatalf("Error getting information on the local node: %s", err.Error())
}

hostname, err := os.Hostname()
if err != nil {
return types.SubmarinerEndpoint{}, fmt.Errorf("error getting hostname: %v", err)
Expand Down Expand Up @@ -72,7 +81,7 @@ func GetLocal(submSpec types.SubmarinerSpecification, privateIP string, gwNode *
},
}

publicIP, err := ipify.GetIp()
publicIP, err := getPublicIP(submSpec, k8sClient, backendConfig)
if err != nil {
return types.SubmarinerEndpoint{}, fmt.Errorf("could not determine public IP: %v", err)
}
Expand Down
23 changes: 19 additions & 4 deletions pkg/endpoint/local_endpoint_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,30 @@ limitations under the License.
package endpoint_test

import (
"os"

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
endpoint "github.com/submariner-io/submariner/pkg/endpoint"

"github.com/submariner-io/submariner/pkg/util"
v1 "k8s.io/api/core/v1"
v1meta "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/fake"

"github.com/submariner-io/submariner/pkg/endpoint"
"github.com/submariner-io/submariner/pkg/types"
)

const testNodeName = "this-node"

var _ = Describe("GetLocal", func() {
var submSpec types.SubmarinerSpecification
var client kubernetes.Interface
var testPrivateIP = util.GetLocalIP()
var node *v1.Node

const (
testPrivateIP = "127.0.0.1"
testUDPPort = "1111"
testUDPPortLabel = "udp-port"
testNATTPortLabel = "natt-discovery-port"
Expand All @@ -49,14 +59,19 @@ var _ = Describe("GetLocal", func() {

node = &v1.Node{
ObjectMeta: v1meta.ObjectMeta{
Name: testNodeName,
Labels: map[string]string{
backendConfigPrefix + testNATTPortLabel: "1234",
backendConfigPrefix + testUDPPortLabel: testUDPPort,
}}}

client = fake.NewSimpleClientset(node)

os.Setenv("NODE_NAME", testNodeName)
})

It("should return a valid SubmarinerEndpoint object", func() {
endpoint, err := endpoint.GetLocal(submSpec, testPrivateIP, node)
endpoint, err := endpoint.GetLocal(submSpec, client)

Expect(err).ToNot(HaveOccurred())
Expect(endpoint.Spec.ClusterID).To(Equal("east"))
Expand All @@ -72,7 +87,7 @@ var _ = Describe("GetLocal", func() {
When("no NAT discovery port label is set on the node", func() {
It("should return a valid SubmarinerEndpoint object", func() {
delete(node.Labels, testNATTPortLabel)
_, err := endpoint.GetLocal(submSpec, testPrivateIP, node)
_, err := endpoint.GetLocal(submSpec, client)
Expect(err).ToNot(HaveOccurred())
})
})
Expand Down
149 changes: 149 additions & 0 deletions pkg/endpoint/public_ip.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
/*
SPDX-License-Identifier: Apache-2.0
Copyright Contributors to the Submariner project.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package endpoint

import (
"context"
"io/ioutil"
"net"
"net/http"
"regexp"
"strings"

"github.com/pkg/errors"
v1 "github.com/submariner-io/submariner/pkg/apis/submariner.io/v1"
"github.com/submariner-io/submariner/pkg/types"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/klog"
)

type publicIPResolverFunction func(clientset kubernetes.Interface, namespace, value string) (string, error)

var publicIPMethods = map[string]publicIPResolverFunction{
v1.API: publicAPI,
v1.IPv4: publicIP,
v1.LoadBalancer: publicLoadBalancerIP,
v1.DNS: publicDNSIP,
}

var IPv4RE = regexp.MustCompile(`(?:\d{1,3}\.){3}\d{1,3}`)

func getPublicIP(submSpec types.SubmarinerSpecification, k8sClient kubernetes.Interface, backendConfig map[string]string) (string, error) {
config, ok := backendConfig[v1.PublicIP]
if !ok {
config = "api:api.ipify.org,api:api.my-ip.io/ip,api:ip4.seeip.org"
}

resolvers := strings.Split(config, ",")
for _, resolver := range resolvers {
resolver = strings.Trim(resolver, " ")
parts := strings.Split(resolver, ":")
if len(parts) != 2 {
return "", errors.Errorf("invalid format for %q label: %q", v1.GatewayConfigLabelPrefix+v1.PublicIP, config)
}

method, ok := publicIPMethods[parts[0]]
if !ok {
return "", errors.Errorf("unknown resolver %q in %q label: %q", parts[0], v1.GatewayConfigLabelPrefix+v1.PublicIP, config)
}

ip, err := method(k8sClient, submSpec.Namespace, parts[1])
if err == nil {
return ip, nil
}

// If this resolved failed we log it, but we fall back to the next one
klog.Errorf("Error resolving public IP with resolver %s : %s", resolver, err.Error())
}

if len(resolvers) > 0 {
return "", errors.Errorf("Unable to resolve public IP by any of the resolver methods: %s", resolvers)
}

return "", nil
}

func publicAPI(clientset kubernetes.Interface, namespace, value string) (string, error) {
url := "https://" + value

//nolint:gosec // we really need to get from a non-predefined const URL
response, err := http.Get(url)
if err != nil {
return "", errors.Wrapf(err, "retrieving public IP from %s", url)
}

defer response.Body.Close()

body, err := ioutil.ReadAll(response.Body)
if err != nil {
return "", errors.Wrapf(err, "reading API response from %s", url)
}

return firstIPv4InString(string(body))
}

func publicIP(clientset kubernetes.Interface, namespace, value string) (string, error) {
return firstIPv4InString(value)
}

func publicLoadBalancerIP(clientset kubernetes.Interface, namespace, loadBalancerName string) (string, error) {
service, err := clientset.CoreV1().Services(namespace).Get(context.TODO(), loadBalancerName, metav1.GetOptions{})
if err != nil {
return "", errors.Wrapf(err, "error getting service %q for the public IP address", loadBalancerName)
}

if len(service.Status.LoadBalancer.Ingress) < 1 {
return "", errors.Errorf("service %q doesn't contain any LoadBalancer ingress", loadBalancerName)
}

ingress := service.Status.LoadBalancer.Ingress[0]
if ingress.IP != "" {
return ingress.IP, nil
}

if ingress.Hostname != "" {
ip, err := publicDNSIP(clientset, namespace, ingress.Hostname)
if err != nil {
return "", errors.Wrapf(err, "error resolving LoadBalancer ingress HostName %q", ingress.Hostname)
}

return ip, nil
}

return "", errors.Errorf("no IP or Hostname for service LoadBalancer %q Ingress", loadBalancerName)
}

func publicDNSIP(clientset kubernetes.Interface, namespace, fqdn string) (string, error) {
ips, err := net.LookupIP(fqdn)
if err != nil {
return "", errors.Wrapf(err, "error resolving DNS hostname %q for public IP", fqdn)
}

return ips[0].String(), nil
}

func firstIPv4InString(body string) (string, error) {
matches := IPv4RE.FindAllString(body, -1)
if len(matches) == 0 {
return "", errors.Errorf("No IPv4 found in: %q", body)
}

return matches[0], nil
}

0 comments on commit ca6d02e

Please sign in to comment.