diff --git a/test/e2e/aw_fixtures.go b/test/e2e/aw_fixtures.go index b52b3bd..196d857 100644 --- a/test/e2e/aw_fixtures.go +++ b/test/e2e/aw_fixtures.go @@ -261,298 +261,6 @@ func createDeploymentAW(ctx context.Context, name string) *arbv1.AppWrapper { return aw } -func createDeploymentAWwith550CPU(ctx context.Context, name string) *arbv1.AppWrapper { - rb := []byte(`{"apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": { - "name": "` + name + `", - "namespace": "test", - "labels": { - "app": "` + name + `" - } - }, - "spec": { - "replicas": 2, - "selector": { - "matchLabels": { - "app": "` + name + `" - } - }, - "template": { - "metadata": { - "labels": { - "app": "` + name + `" - } - }, - "spec": { - "containers": [ - { - "name": "` + name + `", - "image": "quay.io/project-codeflare/echo-server:1.0", - "resources": { - "requests": { - "cpu": "550m" - } - }, - "ports": [ - { - "containerPort": 80 - } - ] - } - ] - } - } - }} `) - var schedSpecMin int32 = 2 - - aw := &arbv1.AppWrapper{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: testNamespace, - }, - Spec: arbv1.AppWrapperSpec{ - Scheduling: arbv1.SchedulingSpec{ - MinAvailable: schedSpecMin, - }, - Resources: arbv1.AppWrapperResources{ - GenericItems: []arbv1.GenericItem{ - { - DoNotUseReplicas: 1, - GenericTemplate: runtime.RawExtension{ - Raw: rb, - }, - }, - }, - }, - }, - } - - err := getClient(ctx).Create(ctx, aw) - Expect(err).NotTo(HaveOccurred()) - - return aw -} - -func createDeploymentAWwith350CPU(ctx context.Context, name string) *arbv1.AppWrapper { - rb := []byte(`{"apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": { - "name": "aw-deployment-2-350cpu", - "namespace": "test", - "labels": { - "app": "aw-deployment-2-350cpu" - } - }, - "spec": { - "replicas": 2, - "selector": { - "matchLabels": { - "app": "aw-deployment-2-350cpu" - } - }, - "template": { - "metadata": { - "labels": { - "app": "aw-deployment-2-350cpu" - } - }, - "spec": { - "containers": [ - { - "name": "aw-deployment-2-350cpu", - "image": "quay.io/project-codeflare/echo-server:1.0", - "resources": { - "requests": { - "cpu": "350m" - } - }, - "ports": [ - { - "containerPort": 80 - } - ] - } - ] - } - } - }} `) - var schedSpecMin int32 = 2 - - aw := &arbv1.AppWrapper{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: testNamespace, - }, - Spec: arbv1.AppWrapperSpec{ - Scheduling: arbv1.SchedulingSpec{ - MinAvailable: schedSpecMin, - }, - Resources: arbv1.AppWrapperResources{ - GenericItems: []arbv1.GenericItem{ - { - DoNotUseReplicas: 1, - GenericTemplate: runtime.RawExtension{ - Raw: rb, - }, - }, - }, - }, - }, - } - - err := getClient(ctx).Create(ctx, aw) - Expect(err).NotTo(HaveOccurred()) - - return aw -} - -func createDeploymentAWwith426CPU(ctx context.Context, name string) *arbv1.AppWrapper { - rb := []byte(`{"apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": { - "name": "` + name + `", - "namespace": "test", - "labels": { - "app": "` + name + `" - } - }, - "spec": { - "replicas": 2, - "selector": { - "matchLabels": { - "app": "` + name + `" - } - }, - "template": { - "metadata": { - "labels": { - "app": "` + name + `" - } - }, - "spec": { - "containers": [ - { - "name": "` + name + `", - "image": "quay.io/project-codeflare/echo-server:1.0", - "resources": { - "requests": { - "cpu": "427m" - } - }, - "ports": [ - { - "containerPort": 80 - } - ] - } - ] - } - } - }} `) - var schedSpecMin int32 = 2 - - aw := &arbv1.AppWrapper{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: testNamespace, - }, - Spec: arbv1.AppWrapperSpec{ - Scheduling: arbv1.SchedulingSpec{ - MinAvailable: schedSpecMin, - }, - Resources: arbv1.AppWrapperResources{ - GenericItems: []arbv1.GenericItem{ - { - DoNotUseReplicas: 1, - GenericTemplate: runtime.RawExtension{ - Raw: rb, - }, - }, - }, - }, - }, - } - - err := getClient(ctx).Create(ctx, aw) - Expect(err).NotTo(HaveOccurred()) - - return aw -} - -func createDeploymentAWwith425CPU(ctx context.Context, name string) *arbv1.AppWrapper { - rb := []byte(`{"apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": { - "name": "aw-deployment-2-425cpu", - "namespace": "test", - "labels": { - "app": "aw-deployment-2-425cpu" - } - }, - "spec": { - "replicas": 2, - "selector": { - "matchLabels": { - "app": "aw-deployment-2-425cpu" - } - }, - "template": { - "metadata": { - "labels": { - "app": "aw-deployment-2-425cpu" - } - }, - "spec": { - "containers": [ - { - "name": "aw-deployment-2-425cpu", - "image": "quay.io/project-codeflare/echo-server:1.0", - "resources": { - "requests": { - "cpu": "425m" - } - }, - "ports": [ - { - "containerPort": 80 - } - ] - } - ] - } - } - }} `) - var schedSpecMin int32 = 2 - - aw := &arbv1.AppWrapper{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: testNamespace, - }, - Spec: arbv1.AppWrapperSpec{ - Scheduling: arbv1.SchedulingSpec{ - MinAvailable: schedSpecMin, - }, - Resources: arbv1.AppWrapperResources{ - GenericItems: []arbv1.GenericItem{ - { - DoNotUseReplicas: 1, - GenericTemplate: runtime.RawExtension{ - Raw: rb, - }, - }, - }, - }, - }, - } - - err := getClient(ctx).Create(ctx, aw) - Expect(err).NotTo(HaveOccurred()) - - return aw -} - func createGenericDeploymentAW(ctx context.Context, name string) *arbv1.AppWrapper { rb := []byte(`{"apiVersion": "apps/v1", "kind": "Deployment", @@ -1132,7 +840,7 @@ func createGenericDeploymentAWWithMultipleItems(ctx context.Context, name string return aw } -func createGenericDeploymentWithCPUAW(ctx context.Context, name string, cpuDemand string, replicas int) *arbv1.AppWrapper { +func createGenericDeploymentWithCPUAW(ctx context.Context, name string, cpuDemand *resource.Quantity, replicas int) *arbv1.AppWrapper { rb := []byte(fmt.Sprintf(`{ "apiVersion": "apps/v1", "kind": "Deployment", @@ -1177,8 +885,6 @@ func createGenericDeploymentWithCPUAW(ctx context.Context, name string, cpuDeman } }} `, name, name, replicas, name, name, name, cpuDemand)) - var schedSpecMin int32 = int32(replicas) - aw := &arbv1.AppWrapper{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -1186,12 +892,15 @@ func createGenericDeploymentWithCPUAW(ctx context.Context, name string, cpuDeman }, Spec: arbv1.AppWrapperSpec{ Scheduling: arbv1.SchedulingSpec{ - MinAvailable: schedSpecMin, + MinAvailable: int32(replicas), }, Resources: arbv1.AppWrapperResources{ GenericItems: []arbv1.GenericItem{ - { - DoNotUseReplicas: 1, + {CustomPodResources: []arbv1.CustomPodResource{ + { + Replicas: int32(replicas), + Requests: map[v1.ResourceName]resource.Quantity{v1.ResourceCPU: *cpuDemand}}, + }, GenericTemplate: runtime.RawExtension{ Raw: rb, }, diff --git a/test/e2e/queue.go b/test/e2e/queue.go index edf7b87..1a5ddea 100644 --- a/test/e2e/queue.go +++ b/test/e2e/queue.go @@ -33,18 +33,20 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" ) +var ctx context.Context + var _ = BeforeSuite(func() { log.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + ctx = extendContextWithClient(context.Background()) + ensureNamespaceExists(ctx) + updateClusterCapacity(ctx) }) var _ = Describe("AppWrapper E2E Tests", func() { - var ctx context.Context var appwrappers []*arbv1.AppWrapper BeforeEach(func() { appwrappers = []*arbv1.AppWrapper{} - ctx = extendContextWithClient(context.Background()) - ensureNamespaceExists(ctx) }) AfterEach(func() { @@ -55,51 +57,50 @@ var _ = Describe("AppWrapper E2E Tests", func() { It("MCAD CPU Accounting Test", func() { fmt.Fprintf(os.Stdout, "[e2e] MCAD CPU Accounting Test - Started.\n") - // This should fill up the worker node and most of the master node - aw := createDeploymentAWwith550CPU(ctx, appendRandomString("aw-deployment-2-550cpu")) - appwrappers = append(appwrappers, aw) + By("Request 55% of cluster CPU") + aw := createGenericDeploymentWithCPUAW(ctx, appendRandomString("aw-deployment-55-percent-cpu"), cpuDemand(0.275), 2) + appwrappers = append(appwrappers, aw) err := waitAWPodsReady(ctx, aw) - Expect(err).NotTo(HaveOccurred(), "Ready pods are expected for app wrapper: aw-deployment-2-550cpu") + Expect(err).NotTo(HaveOccurred(), "Ready pods are expected for app wrapper: aw-deployment-55-percent-cpu") - // This should fill up the master node - aw2 := createDeploymentAWwith350CPU(ctx, appendRandomString("aw-deployment-2-350cpu")) + By("Request 30% of cluster CPU") + aw2 := createGenericDeploymentWithCPUAW(ctx, appendRandomString("aw-deployment-30-percent-cpu"), cpuDemand(0.15), 2) appwrappers = append(appwrappers, aw2) - - // Using quiet mode due to creating of pods in earlier step. err = waitAWReadyQuiet(ctx, aw2) - Expect(err).NotTo(HaveOccurred(), "Ready pods are expected for app wrapper: aw-deployment-2-350cpu") - fmt.Fprintf(os.Stdout, "[e2e] MCAD CPU Accounting Test - Completed. Awaiting app wrapper cleanup...\n") + Expect(err).NotTo(HaveOccurred(), "Ready pods are expected for app wrapper:aw-deployment-30-percent-cpu") }) - /* - TODO: DAVE -- Test disabled because it is too fragile -- redo with % of capacity calculation - It("MCAD CPU Preemption Test", func() { - fmt.Fprintf(os.Stdout, "[e2e] MCAD CPU Preemption Test - Started.\n") + It("MCAD CPU Queueing Test", func() { + fmt.Fprintf(os.Stdout, "[e2e] MCAD CPU Queueing Test - Started.\n") - // This should fill up the worker node and most of the master node - aw := createDeploymentAWwith550CPU(ctx, appendRandomString("aw-deployment-2-550cpu")) - appwrappers = append(appwrappers, aw) - err := waitAWPodsReady(ctx, aw) - Expect(err).NotTo(HaveOccurred()) - - // This should not fit on cluster - aw2 := createDeploymentAWwith426CPU(ctx, appendRandomString("aw-deployment-2-426cpu")) - appwrappers = append(appwrappers, aw2) - err = waitAWAnyPodsExists(ctx, aw2) - // With improved accounting, no pods will be spawned - Expect(err).To(HaveOccurred()) + By("Request 55% of cluster CPU") + aw := createGenericDeploymentWithCPUAW(ctx, appendRandomString("aw-deployment-55-percent-cpu"), cpuDemand(0.275), 2) + appwrappers = append(appwrappers, aw) + err := waitAWPodsReady(ctx, aw) + Expect(err).NotTo(HaveOccurred(), "Ready pods are expected for app wrapper: aw-deployment-55-percent-cpu") - // This should fit on cluster, initially queued because of aw2 above but should eventually - // run after prevention of aw2 above. - aw3 := createDeploymentAWwith425CPU(ctx, "aw-deployment-2-425cpu") - appwrappers = append(appwrappers, aw3) + By("Request 50% of cluster CPU (will not fit; should not be dispatched)") + aw2 := createGenericDeploymentWithCPUAW(ctx, appendRandomString("aw-deployment-50-percent-cpu"), cpuDemand(0.25), 2) + appwrappers = append(appwrappers, aw2) + err = waitAWAnyPodsExists(ctx, aw2) + Expect(err).To(HaveOccurred(), "Ready pods were not expected for app wrapper: aw-deployment-50-percent-cpu") - // Since preemption takes some time, increasing timeout wait time to 2 minutes - err = waitAWPodsExists(ctx, aw3, 2*time.Minute) - Expect(err).NotTo(HaveOccurred(), "Expecting pods for app wrapper : aw-deployment-2-425cpu") - }) - */ + By("Request 30% of cluster CPU") + aw3 := createGenericDeploymentWithCPUAW(ctx, appendRandomString("aw-deployment-30-percent-cpu"), cpuDemand(0.15), 2) + appwrappers = append(appwrappers, aw3) + err = waitAWReadyQuiet(ctx, aw3) + Expect(err).NotTo(HaveOccurred(), "Ready pods are expected for app wrapper:aw-deployment-30-percent-cpu") + + By("Free resources by deleting 55% of cluster AppWrapper") + err = deleteAppWrapper(ctx, aw.Name, aw.Namespace) + appwrappers = []*arbv1.AppWrapper{aw2, aw3} + Expect(err).NotTo(HaveOccurred(), "Should have been able to delete an the initial AppWrapper") + + By("Wait for queued AppWrapper to finally be dispatched") + err = waitAWReadyQuiet(ctx, aw2) + Expect(err).NotTo(HaveOccurred(), "Ready pods are expected for app wrapper: aw-deployment-50-percent-cpu") + }) /* * TODO: Dave DISABLED BECASUSE V2 doesn't support exponential backoff of requeuing time @@ -137,7 +138,6 @@ var _ = Describe("AppWrapper E2E Tests", func() { aw := createStatefulSetAW(ctx, "aw-statefulset-2") appwrappers = append(appwrappers, aw) - err := waitAWPodsReady(ctx, aw) Expect(err).NotTo(HaveOccurred()) }) @@ -147,7 +147,6 @@ var _ = Describe("AppWrapper E2E Tests", func() { aw := createGenericStatefulSetAW(ctx, "aw-generic-statefulset-2") appwrappers = append(appwrappers, aw) - err := waitAWPodsReady(ctx, aw) Expect(err).NotTo(HaveOccurred()) }) @@ -157,8 +156,6 @@ var _ = Describe("AppWrapper E2E Tests", func() { aw := createDeploymentAW(ctx, "aw-deployment-3") appwrappers = append(appwrappers, aw) - - fmt.Fprintf(GinkgoWriter, "[e2e] Awaiting %d pods running for AW %s.\n", aw.Spec.Scheduling.MinAvailable, aw.Name) err := waitAWPodsReady(ctx, aw) Expect(err).NotTo(HaveOccurred()) }) @@ -168,7 +165,6 @@ var _ = Describe("AppWrapper E2E Tests", func() { aw := createGenericDeploymentAW(ctx, "aw-generic-deployment-3") appwrappers = append(appwrappers, aw) - err := waitAWPodsReady(ctx, aw) Expect(err).NotTo(HaveOccurred()) }) @@ -178,7 +174,6 @@ var _ = Describe("AppWrapper E2E Tests", func() { aw := createPodTemplateAW(ctx, "aw-podtemplate-2") appwrappers = append(appwrappers, aw) - err := waitAWPodsReady(ctx, aw) Expect(err).NotTo(HaveOccurred()) }) @@ -188,7 +183,6 @@ var _ = Describe("AppWrapper E2E Tests", func() { aw := createGenericPodAW(ctx, "aw-generic-pod-1") appwrappers = append(appwrappers, aw) - err := waitAWPodsReady(ctx, aw) Expect(err).NotTo(HaveOccurred()) }) @@ -216,7 +210,6 @@ var _ = Describe("AppWrapper E2E Tests", func() { aw := createBadPodTemplateAW(ctx, "aw-bad-podtemplate-2") appwrappers = append(appwrappers, aw) - err := waitAWPodsExists(ctx, aw, 30*time.Second) Expect(err).To(HaveOccurred()) }) @@ -236,7 +229,6 @@ var _ = Describe("AppWrapper E2E Tests", func() { aw := createBadGenericPodAW(ctx, "aw-bad-generic-pod-1") appwrappers = append(appwrappers, aw) - err := waitAWPodsCompleted(ctx, aw, 10*time.Second) Expect(err).To(HaveOccurred()) @@ -247,7 +239,6 @@ var _ = Describe("AppWrapper E2E Tests", func() { aw := createBadGenericItemAW(ctx, "aw-bad-generic-item-1") appwrappers = append(appwrappers, aw) - err := waitAWPodsCompleted(ctx, aw, 10*time.Second) Expect(err).To(HaveOccurred()) @@ -477,14 +468,13 @@ var _ = Describe("AppWrapper E2E Tests", func() { appwrappers = append(appwrappers, aw) fmt.Fprintf(GinkgoWriter, "The error is: %v", err1) Expect(err1).To(HaveOccurred(), "Expecting for pods not to be ready for app wrapper: aw-test-job-with-comp-44") - fmt.Fprintf(os.Stdout, "[e2e] MCAD GenericItem Without Status Test - Completed.\n") - }) It("MCAD Job Completion No-requeue Test", Label("slow"), func() { fmt.Fprintf(os.Stdout, "[e2e] MCAD Job Completion No-requeue Test - Started.\n") aw := createGenericJobAWWithScheduleSpec(ctx, "aw-test-job-with-scheduling-spec") + appwrappers = append(appwrappers, aw) err1 := waitAWPodsReady(ctx, aw) Expect(err1).NotTo(HaveOccurred(), "Waiting for pods to be ready") err2 := waitAWPodsCompleted(ctx, aw, 90*time.Second) @@ -494,10 +484,6 @@ var _ = Describe("AppWrapper E2E Tests", func() { // which SHOULD NOT happen because the job is done err3 := waitAWPodsNotCompleted(ctx, aw) Expect(err3).To(HaveOccurred(), "Waiting for pods not to be completed") - - appwrappers = append(appwrappers, aw) - fmt.Fprintf(os.Stdout, "[e2e] MCAD Job Completion No-requeue Test - Completed.\n") - }) /* TODO: DAVE: Status.State @@ -513,27 +499,6 @@ var _ = Describe("AppWrapper E2E Tests", func() { }) */ - /* - TODO: DAVE -- Test disabled because it is too fragile -- redo with % of capacity calculation - It("MCAD CPU Accounting Queuing Test", func() { - fmt.Fprintf(os.Stdout, "[e2e] MCAD CPU Accounting Queuing Test - Started.\n") - - // This should fill up the worker node and most of the master node - aw := createDeploymentAWwith550CPU(ctx, appendRandomString("aw-deployment-2-550cpu")) - appwrappers = append(appwrappers, aw) - err := waitAWPodsReady(ctx, aw) - Expect(err).NotTo(HaveOccurred(), "Waiting for pods to be ready for app wrapper: aw-deployment-2-550cpu") - - // This should not fit on cluster - // there may be a false positive dispatch which will cause MCAD to requeue AW - aw2 := createDeploymentAWwith426CPU(ctx, appendRandomString("aw-deployment-2-426cpu")) - appwrappers = append(appwrappers, aw2) - - err = waitAWPodsReady(ctx, aw2) - Expect(err).To(HaveOccurred(), "No pods for app wrapper `aw-deployment-2-426cpu` are expected.") - }) - */ - /* TODO: DAVE: Status.State It("MCAD Deployment RunningHoldCompletion Test", func() { fmt.Fprintf(os.Stdout, "[e2e] MCAD Deployment RunningHoldCompletion Test - Started.\n") @@ -568,9 +533,10 @@ var _ = Describe("AppWrapper E2E Tests", func() { const ( awCount = 50 reportingInterval = 10 - cpuDemand = "5m" + cpuPercent = 0.005 ) + cpuDemand := cpuDemand(cpuPercent) replicas := 2 modDivisor := int(awCount / reportingInterval) for i := 0; i < awCount; i++ { diff --git a/test/e2e/util.go b/test/e2e/util.go index ec51234..d2442dc 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -18,6 +18,7 @@ package e2e import ( "context" + "encoding/json" "fmt" "math/rand" "strings" @@ -27,6 +28,7 @@ import ( "github.com/onsi/gomega" . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" @@ -37,6 +39,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" arbv1 "github.com/project-codeflare/mcad/api/v1beta1" + arcont "github.com/project-codeflare/mcad/internal/controller" ) const testNamespace = "test" @@ -45,6 +48,7 @@ var ninetySeconds = 90 * time.Second var threeMinutes = 180 * time.Second var tenMinutes = 600 * time.Second var threeHundredSeconds = 300 * time.Second +var clusterCapacity v1.ResourceList = v1.ResourceList{} type myKey struct { key string @@ -73,6 +77,118 @@ func ensureNamespaceExists(ctx context.Context) { Expect(client.IgnoreAlreadyExists(err)).NotTo(HaveOccurred()) } +// Compute available cluster capacity to allow tests to scale their resources appropriately. +// The code is a simplification of AppWrapperReconciler.computeCapacity and intended to be run +// in BeforeSuite methods (thus it is not necessary to filter out AppWrapper-owned pods) +func updateClusterCapacity(ctx context.Context) { + kc := getClient(ctx) + capacity := arcont.Weights{} + // add allocatable capacity for each schedulable node + nodes := &v1.NodeList{} + err := kc.List(ctx, nodes) + Expect(err).NotTo(HaveOccurred()) + +LOOP: + for _, node := range nodes.Items { + // skip unschedulable nodes + if node.Spec.Unschedulable { + continue + } + for _, taint := range node.Spec.Taints { + if taint.Effect == v1.TaintEffectNoSchedule || taint.Effect == v1.TaintEffectNoExecute { + continue LOOP + } + } + // add allocatable capacity on the node + capacity.Add(arcont.NewWeights(node.Status.Allocatable)) + } + // subtract requests from non-terminated pods + pods := &v1.PodList{} + err = kc.List(ctx, pods) + Expect(err).NotTo(HaveOccurred()) + for _, pod := range pods.Items { + if pod.Status.Phase != v1.PodFailed && pod.Status.Phase != v1.PodSucceeded { + capacity.Sub(arcont.NewWeightsForPod(&pod)) + } + } + + clusterCapacity = capacity.AsResources() + + t, _ := json.Marshal(clusterCapacity) + fmt.Fprintf(GinkgoWriter, "Computed cluster capacity: %v\n", string(t)) +} + +func cpuDemand(fractionOfCluster float64) *resource.Quantity { + clusterCPU := clusterCapacity[v1.ResourceCPU] + milliDemand := int64(float64(clusterCPU.MilliValue()) * fractionOfCluster) + return resource.NewMilliQuantity(milliDemand, resource.DecimalSI) +} + +func cleanupTestObjectsPtr(ctx context.Context, appwrappersPtr *[]*arbv1.AppWrapper) { + cleanupTestObjectsPtrVerbose(ctx, appwrappersPtr, true) +} + +func cleanupTestObjectsPtrVerbose(ctx context.Context, appwrappersPtr *[]*arbv1.AppWrapper, verbose bool) { + if appwrappersPtr == nil { + fmt.Fprintf(GinkgoWriter, "[cleanupTestObjectsPtr] No AppWrappers to cleanup.\n") + } else { + cleanupTestObjects(ctx, *appwrappersPtr) + } +} + +func cleanupTestObjects(ctx context.Context, appwrappers []*arbv1.AppWrapper) { + cleanupTestObjectsVerbose(ctx, appwrappers, true) +} + +func cleanupTestObjectsVerbose(ctx context.Context, appwrappers []*arbv1.AppWrapper, verbose bool) { + if appwrappers == nil { + fmt.Fprintf(GinkgoWriter, "[cleanupTestObjects] No AppWrappers to cleanup.\n") + return + } + + for _, aw := range appwrappers { + pods := getPodsOfAppWrapper(ctx, aw) + awNamespace := aw.Namespace + awName := aw.Name + fmt.Fprintf(GinkgoWriter, "[cleanupTestObjects] Deleting AW %s.\n", aw.Name) + err := deleteAppWrapper(ctx, aw.Name, aw.Namespace) + Expect(err).NotTo(HaveOccurred()) + + // Wait for the pods of the deleted the appwrapper to be destroyed + for _, pod := range pods { + fmt.Fprintf(GinkgoWriter, "[cleanupTestObjects] Awaiting pod %s/%s to be deleted for AW %s.\n", + pod.Namespace, pod.Name, aw.Name) + } + err = waitAWPodsDeleted(ctx, awNamespace, awName, pods) + + // Final check to see if pod exists + if err != nil { + var podsStillExisting []*v1.Pod + for _, pod := range pods { + podExist := &v1.Pod{} + err = getClient(ctx).Get(ctx, client.ObjectKey{Namespace: pod.Namespace, Name: pod.Name}, podExist) + if err != nil { + fmt.Fprintf(GinkgoWriter, "[cleanupTestObjects] Found pod %s/%s %s, not completedly deleted for AW %s.\n", podExist.Namespace, podExist.Name, podExist.Status.Phase, aw.Name) + podsStillExisting = append(podsStillExisting, podExist) + } + } + if len(podsStillExisting) > 0 { + err = waitAWPodsDeleted(ctx, awNamespace, awName, podsStillExisting) + } + } + Expect(err).NotTo(HaveOccurred()) + } +} + +func deleteAppWrapper(ctx context.Context, name string, namespace string) error { + foreground := metav1.DeletePropagationForeground + aw := &arbv1.AppWrapper{ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }} + return getClient(ctx).Delete(ctx, aw, &client.DeleteOptions{PropagationPolicy: &foreground}) +} + func anyPodsExist(ctx context.Context, awNamespace string, awName string) wait.ConditionFunc { return func() (bool, error) { podList := &v1.PodList{} @@ -149,62 +265,6 @@ func podPhase(ctx context.Context, awNamespace string, awName string, pods []*v1 } } -func cleanupTestObjectsPtr(ctx context.Context, appwrappersPtr *[]*arbv1.AppWrapper) { - cleanupTestObjectsPtrVerbose(ctx, appwrappersPtr, true) -} - -func cleanupTestObjectsPtrVerbose(ctx context.Context, appwrappersPtr *[]*arbv1.AppWrapper, verbose bool) { - if appwrappersPtr == nil { - fmt.Fprintf(GinkgoWriter, "[cleanupTestObjectsPtr] No AppWrappers to cleanup.\n") - } else { - cleanupTestObjects(ctx, *appwrappersPtr) - } -} - -func cleanupTestObjects(ctx context.Context, appwrappers []*arbv1.AppWrapper) { - cleanupTestObjectsVerbose(ctx, appwrappers, true) -} - -func cleanupTestObjectsVerbose(ctx context.Context, appwrappers []*arbv1.AppWrapper, verbose bool) { - if appwrappers == nil { - fmt.Fprintf(GinkgoWriter, "[cleanupTestObjects] No AppWrappers to cleanup.\n") - return - } - - for _, aw := range appwrappers { - pods := getPodsOfAppWrapper(ctx, aw) - awNamespace := aw.Namespace - awName := aw.Name - fmt.Fprintf(GinkgoWriter, "[cleanupTestObjects] Deleting AW %s.\n", aw.Name) - err := deleteAppWrapper(ctx, aw.Name, aw.Namespace) - Expect(err).NotTo(HaveOccurred()) - - // Wait for the pods of the deleted the appwrapper to be destroyed - for _, pod := range pods { - fmt.Fprintf(GinkgoWriter, "[cleanupTestObjects] Awaiting pod %s/%s to be deleted for AW %s.\n", - pod.Namespace, pod.Name, aw.Name) - } - err = waitAWPodsDeleted(ctx, awNamespace, awName, pods) - - // Final check to see if pod exists - if err != nil { - var podsStillExisting []*v1.Pod - for _, pod := range pods { - podExist := &v1.Pod{} - err = getClient(ctx).Get(ctx, client.ObjectKey{Namespace: pod.Namespace, Name: pod.Name}, podExist) - if err != nil { - fmt.Fprintf(GinkgoWriter, "[cleanupTestObjects] Found pod %s/%s %s, not completedly deleted for AW %s.\n", podExist.Namespace, podExist.Name, podExist.Status.Phase, aw.Name) - podsStillExisting = append(podsStillExisting, podExist) - } - } - if len(podsStillExisting) > 0 { - err = waitAWPodsDeleted(ctx, awNamespace, awName, podsStillExisting) - } - } - Expect(err).NotTo(HaveOccurred()) - } -} - func awPodPhase(ctx context.Context, aw *arbv1.AppWrapper, phase []v1.PodPhase, taskNum int, quite bool) wait.ConditionFunc { return func() (bool, error) { defer GinkgoRecover() @@ -268,34 +328,6 @@ func awPodPhase(ctx context.Context, aw *arbv1.AppWrapper, phase []v1.PodPhase, } } -func awNamespacePhase(ctx context.Context, aw *arbv1.AppWrapper, phase []v1.NamespacePhase) wait.ConditionFunc { - return func() (bool, error) { - awIgnored := &arbv1.AppWrapper{} - err := getClient(ctx).Get(ctx, client.ObjectKey{Namespace: aw.Namespace, Name: aw.Name}, awIgnored) // TODO: Do we actually need to do this Get? - Expect(err).NotTo(HaveOccurred()) - - namespaces := &v1.NamespaceList{} - err = getClient(ctx).List(ctx, namespaces) - Expect(err).NotTo(HaveOccurred()) - - readyTaskNum := 0 - for _, namespace := range namespaces.Items { - if awns, found := namespace.Labels["appwrapper.mcad.ibm.com"]; !found || awns != aw.Name { - continue - } - - for _, p := range phase { - if namespace.Status.Phase == p { - readyTaskNum++ - break - } - } - } - - return 0 < readyTaskNum, nil - } -} - func waitAWPodsReady(ctx context.Context, aw *arbv1.AppWrapper) error { return waitAWPodsReadyEx(ctx, aw, ninetySeconds, int(aw.Spec.Scheduling.MinAvailable), false) } @@ -370,16 +402,6 @@ func waitAWPodsTerminatedExVerbose(ctx context.Context, namespace string, name s []v1.PodPhase{v1.PodRunning, v1.PodSucceeded, v1.PodUnknown, v1.PodFailed, v1.PodPending}, taskNum)) } -func deleteAppWrapper(ctx context.Context, name string, namespace string) error { - foreground := metav1.DeletePropagationForeground - aw := &arbv1.AppWrapper{ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }} - return getClient(ctx).Delete(ctx, aw, &client.DeleteOptions{PropagationPolicy: &foreground}) - -} - func getPodsOfAppWrapper(ctx context.Context, aw *arbv1.AppWrapper) []*v1.Pod { awIgnored := &arbv1.AppWrapper{} err := getClient(ctx).Get(ctx, client.ObjectKeyFromObject(aw), awIgnored) // TODO: Do we actually need to do this Get?