Do not append the port to authority (#12350)

* Do not append the port to authority

* Clean up code

* Call SplitHostPort only when domain has a port

test/e2e/grpc_test.go

@@ -73,19 +73,19 @@ func dial(ctx *TestContext, host, domain string) (*grpc.ClientConn, error) {
 	if !hasPort(host) {
 		host = net.JoinHostPort(host, defaultPort)
 	}
-	if !hasPort(domain) {
-		domain = net.JoinHostPort(domain, defaultPort)
+	if hasPort(domain) {
+		var err error
+		domain, _, err = net.SplitHostPort(domain)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	secureOpt := grpc.WithInsecure()
 	if test.ServingFlags.HTTPS {
 		tlsConfig := test.TLSClientConfig(context.Background(), ctx.t.Logf, ctx.clients)
 		// Set ServerName for pseudo hostname with TLS.
-		var err error
-		tlsConfig.ServerName, _, err = net.SplitHostPort(domain)
-		if err != nil {
-			return nil, err
-		}
+		tlsConfig.ServerName = domain
 		secureOpt = grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))
 	}