End to end tests for GameServer Pod Deletion

Context: googleforgames#678

Wrote tests to confirm that deletion of a GameServer's backing Pod would
move a GameServer to `Unhealthy`.

test/e2e/fleet_test.go

@@ -29,6 +29,7 @@ import (
 	v1betaext "k8s.io/api/extensions/v1beta1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -799,6 +800,52 @@ func TestUpdateFleetScheduling(t *testing.T) {
 		})
 }
 
+// TestFleetRecreateGameServerOnPodDeletion ensure that if a pod from a GameServer
+// is deleted, the GameServer is deleted and replaced.
+func TestFleetRecreateGameServerOnPodDeletion(t *testing.T) {
+	t.Parallel()
+
+	alpha1 := framework.AgonesClient.StableV1alpha1()
+	flt := defaultFleet()
+	flt.Spec.Replicas = 1
+
+	flt, err := alpha1.Fleets(defaultNs).Create(flt)
+	podClient := framework.KubeClient.CoreV1().Pods(defaultNs)
+
+	if assert.Nil(t, err) {
+		defer alpha1.Fleets(defaultNs).Delete(flt.ObjectMeta.Name, nil) // nolint:errcheck
+	}
+
+	framework.WaitForFleetCondition(t, flt, e2e.FleetReadyCount(flt.Spec.Replicas))
+
+	selector := labels.SelectorFromSet(labels.Set{v1alpha1.FleetNameLabel: flt.ObjectMeta.Name})
+	list, err := alpha1.GameServers(defaultNs).List(metav1.ListOptions{LabelSelector: selector.String()})
+	assert.NoError(t, err)
+
+	assert.Len(t, list.Items, 1)
+	gs := list.Items[0]
+	pod, err := podClient.Get(gs.ObjectMeta.Name, metav1.GetOptions{})
+	assert.NoError(t, err)
+
+	assert.True(t, metav1.IsControlledBy(pod, &gs))
+
+	err = podClient.Delete(pod.ObjectMeta.Name, nil)
+	assert.NoError(t, err)
+
+	err = wait.Poll(time.Second, 30*time.Second, func() (done bool, err error) {
+		_, err = alpha1.GameServers(defaultNs).Get(gs.ObjectMeta.Name, metav1.GetOptions{})
+
+		if err != nil && k8serrors.IsNotFound(err) {
+			return true, nil
+		}
+
+		return false, err
+	})
+	assert.NoError(t, err)
+
+	framework.WaitForFleetCondition(t, flt, e2e.FleetReadyCount(flt.Spec.Replicas))
+}
+
 // Counts the number of gameservers with the specified scheduling strategy in a fleet
 func countFleetScheduling(gsList []v1alpha1.GameServer, scheduling v1alpha1.SchedulingStrategy) int {
 	count := 0

test/e2e/gameserver_test.go

@@ -18,13 +18,12 @@ import (
 	"testing"
 	"time"
 
-	"k8s.io/apimachinery/pkg/api/resource"
-
 	"agones.dev/agones/pkg/apis/stable"
 	"agones.dev/agones/pkg/apis/stable/v1alpha1"
 	e2eframework "agones.dev/agones/test/e2e/framework"
 	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 )
@@ -179,6 +178,32 @@ func TestUnhealthyGameServersWithoutFreePorts(t *testing.T) {
 	_, err = framework.WaitForGameServerState(newGs, v1alpha1.GameServerStateUnhealthy, 10*time.Second)
 	assert.Nil(t, err)
 }
+
+func TestGameServerUnhealthyAfterDeletingPod(t *testing.T) {
+	t.Parallel()
+	gs := defaultGameServer()
+	readyGs, err := framework.CreateGameServerAndWaitUntilReady(defaultNs, gs)
+	if err != nil {
+		t.Fatalf("Could not get a GameServer ready: %v", err)
+	}
+
+	gsClient := framework.AgonesClient.StableV1alpha1().GameServers(defaultNs)
+	podClient := framework.KubeClient.CoreV1().Pods(defaultNs)
+
+	defer gsClient.Delete(readyGs.ObjectMeta.Name, nil) // nolint: errcheck
+
+	pod, err := podClient.Get(readyGs.ObjectMeta.Name, metav1.GetOptions{})
+	assert.NoError(t, err)
+
+	assert.True(t, metav1.IsControlledBy(pod, readyGs))
+
+	err = podClient.Delete(pod.ObjectMeta.Name, nil)
+	assert.NoError(t, err)
+
+	_, err = framework.WaitForGameServerState(readyGs, v1alpha1.GameServerStateUnhealthy, 10*time.Second)
+	assert.NoError(t, err)
+}
+
 func TestDevelopmentGameServerLifecycle(t *testing.T) {
 	t.Parallel()
 	gs := &v1alpha1.GameServer{