Skip to content

Commit c63277e

Browse files
staeblerkibbles-n-bytes
staebler
authored and
kibbles-n-bytes
committed
Add unit tests verifying deleting a resource with an on-going operation or in orphan mitigation. (openshift#1490)
1 parent 4ecca16 commit c63277e

File tree

2 files changed

+358
-7
lines changed

2 files changed

+358
-7
lines changed

‎pkg/controller/controller_binding_test.go

Lines changed: 215 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2440,14 +2440,23 @@ func TestReconcileBindingWithOrphanMitigationReconciliationRetryTimeOut(t *testi
24402440
ExternalID: testServiceBindingGUID,
24412441
SecretName: testServiceBindingSecretName,
24422442
},
2443+
Status: v1beta1.ServiceBindingStatus{
2444+
Conditions: []v1beta1.ServiceBindingCondition{
2445+
{
2446+
Type: v1beta1.ServiceBindingConditionFailed,
2447+
Status: v1beta1.ConditionTrue,
2448+
Reason: "reason-orphan-mitigation-began",
2449+
},
2450+
},
2451+
},
24432452
}
2444-
startTime := metav1.NewTime(time.Now().Add(-1 * time.Hour))
2453+
startTime := metav1.NewTime(time.Now().Add(-7 * 24 * time.Hour))
24452454
binding.Status.CurrentOperation = v1beta1.ServiceBindingOperationBind
24462455
binding.Status.OperationStartTime = &startTime
24472456
binding.Status.OrphanMitigationInProgress = true
24482457

2449-
if err := testController.reconcileServiceBinding(binding); err == nil {
2450-
t.Fatal("reconciliation shouldn't fully complete due to timeout error")
2458+
if err := testController.reconcileServiceBinding(binding); err != nil {
2459+
t.Fatalf("reconciliation should complete since the retry duration has elapsed: %v", err)
24512460
}
24522461
kubeActions := fakeKubeClient.Actions()
24532462
assertNumberOfActions(t, kubeActions, 1)
@@ -2472,14 +2481,213 @@ func TestReconcileBindingWithOrphanMitigationReconciliationRetryTimeOut(t *testi
24722481
assertNumberOfActions(t, actions, 1)
24732482

24742483
updatedServiceBinding := assertUpdateStatus(t, actions[0], binding).(*v1beta1.ServiceBinding)
2475-
assertServiceBindingCondition(t, updatedServiceBinding, v1beta1.ServiceBindingConditionReady, v1beta1.ConditionUnknown)
2484+
assertServiceBindingRequestFailingError(t, updatedServiceBinding, v1beta1.ServiceBindingOperationUnbind, errorUnbindCallReason, "reason-orphan-mitigation-began", binding)
2485+
assertServiceBindingOrphanMitigationSet(t, updatedServiceBinding, false)
2486+
2487+
expectedEventPrefixes := []string{
2488+
corev1.EventTypeWarning + " " + errorUnbindCallReason,
2489+
corev1.EventTypeWarning + " " + errorReconciliationRetryTimeoutReason,
2490+
}
2491+
events := getRecordedEvents(testController)
2492+
assertNumEvents(t, events, len(expectedEventPrefixes))
2493+
2494+
for i, e := range expectedEventPrefixes {
2495+
a := events[i]
2496+
if !strings.HasPrefix(a, e) {
2497+
t.Fatalf("Received unexpected event:\n expected prefix: %v\n got: %v", e, a)
2498+
}
2499+
}
2500+
}
2501+
2502+
// TestReconcileServiceBindingDeleteDuringOngoingOperation tests deleting a
2503+
// binding that has an on-going operation.
2504+
func TestReconcileServiceBindingDeleteDuringOngoingOperation(t *testing.T) {
2505+
fakeKubeClient, fakeCatalogClient, fakeClusterServiceBrokerClient, testController, sharedInformers := newTestController(t, fakeosb.FakeClientConfiguration{
2506+
UnbindReaction: &fakeosb.UnbindReaction{},
2507+
})
2508+
2509+
sharedInformers.ClusterServiceBrokers().Informer().GetStore().Add(getTestClusterServiceBroker())
2510+
sharedInformers.ClusterServiceClasses().Informer().GetStore().Add(getTestClusterServiceClass())
2511+
sharedInformers.ServiceInstances().Informer().GetStore().Add(getTestServiceInstanceWithRefs())
2512+
sharedInformers.ClusterServicePlans().Informer().GetStore().Add(getTestClusterServicePlan())
2513+
2514+
startTime := metav1.NewTime(time.Now().Add(-1 * time.Hour))
2515+
binding := &v1beta1.ServiceBinding{
2516+
ObjectMeta: metav1.ObjectMeta{
2517+
Name: testServiceBindingName,
2518+
Namespace: testNamespace,
2519+
DeletionTimestamp: &metav1.Time{},
2520+
Finalizers: []string{v1beta1.FinalizerServiceCatalog},
2521+
},
2522+
Spec: v1beta1.ServiceBindingSpec{
2523+
ServiceInstanceRef: v1beta1.LocalObjectReference{Name: testServiceInstanceName},
2524+
ExternalID: testServiceBindingGUID,
2525+
SecretName: testServiceBindingSecretName,
2526+
},
2527+
Status: v1beta1.ServiceBindingStatus{
2528+
CurrentOperation: v1beta1.ServiceBindingOperationBind,
2529+
OperationStartTime: &startTime,
2530+
},
2531+
}
2532+
2533+
fakeCatalogClient.AddReactor("get", "servicebindings", func(action clientgotesting.Action) (bool, runtime.Object, error) {
2534+
return true, binding, nil
2535+
})
2536+
2537+
timeOfReconciliation := metav1.Now()
2538+
2539+
err := testController.reconcileServiceBinding(binding)
2540+
if err != nil {
2541+
t.Fatalf("%v", err)
2542+
}
2543+
2544+
brokerActions := fakeClusterServiceBrokerClient.Actions()
2545+
assertNumberOfClusterServiceBrokerActions(t, brokerActions, 1)
2546+
assertUnbind(t, brokerActions[0], &osb.UnbindRequest{
2547+
BindingID: testServiceBindingGUID,
2548+
InstanceID: testServiceInstanceGUID,
2549+
ServiceID: testClusterServiceClassGUID,
2550+
PlanID: testClusterServicePlanGUID,
2551+
})
2552+
2553+
kubeActions := fakeKubeClient.Actions()
2554+
// The action should be deleting the secret
2555+
assertNumberOfActions(t, kubeActions, 1)
2556+
2557+
deleteAction := kubeActions[0].(clientgotesting.DeleteActionImpl)
2558+
if e, a := "delete", deleteAction.GetVerb(); e != a {
2559+
t.Fatalf("Unexpected verb on kubeActions[1]; expected %v, got %v", e, a)
2560+
}
2561+
2562+
if e, a := binding.Spec.SecretName, deleteAction.Name; e != a {
2563+
t.Fatalf("Unexpected name of secret: expected %v, got %v", e, a)
2564+
}
2565+
2566+
actions := fakeCatalogClient.Actions()
2567+
// The actions should be:
2568+
// 0. Updating the current operation
2569+
// 1. Updating the ready condition
2570+
assertNumberOfActions(t, actions, 2)
2571+
2572+
updatedServiceBinding := assertUpdateStatus(t, actions[0], binding).(*v1beta1.ServiceBinding)
2573+
assertServiceBindingOperationInProgress(t, updatedServiceBinding, v1beta1.ServiceBindingOperationUnbind, binding)
2574+
assertServiceBindingOrphanMitigationSet(t, updatedServiceBinding, false)
2575+
2576+
// Verify that the operation start time was reset to Now
2577+
if updatedServiceBinding.Status.OperationStartTime.Before(&timeOfReconciliation) {
2578+
t.Fatalf(
2579+
"OperationStartTime should not be before the time that the reconciliation started. OperationStartTime=%v. timeOfReconciliation=%v",
2580+
updatedServiceBinding.Status.OperationStartTime,
2581+
timeOfReconciliation,
2582+
)
2583+
}
2584+
2585+
updatedServiceBinding = assertUpdateStatus(t, actions[1], binding).(*v1beta1.ServiceBinding)
2586+
assertServiceBindingOperationSuccess(t, updatedServiceBinding, v1beta1.ServiceBindingOperationUnbind, binding)
2587+
assertServiceBindingOrphanMitigationSet(t, updatedServiceBinding, false)
24762588

2477-
assertServiceBindingOrphanMitigationSet(t, updatedServiceBinding, true)
24782589
events := getRecordedEvents(testController)
24792590
assertNumEvents(t, events, 1)
24802591

2481-
expectedEvent := corev1.EventTypeWarning + " " + errorUnbindCallReason + " " + "Error unbinding from ServiceInstance \"test-ns/test-instance\" of ClusterServiceClass (K8S: \"SCGUID\" ExternalName: \"test-serviceclass\") at ClusterServiceBroker \"test-broker\": timed out"
2592+
expectedEvent := corev1.EventTypeNormal + " " + successUnboundReason + " " + "This binding was deleted successfully"
24822593
if e, a := expectedEvent, events[0]; e != a {
2483-
t.Fatalf("Received unexpected event, %s", expectedGot(e, a))
2594+
t.Fatalf("Received unexpected event: %v", a)
2595+
}
2596+
}
2597+
2598+
// TestReconcileServiceBindingDeleteDuringOrphanMitigation tests deleting a
2599+
// binding that is undergoing orphan mitigation
2600+
func TestReconcileServiceBindingDeleteDuringOrphanMitigation(t *testing.T) {
2601+
fakeKubeClient, fakeCatalogClient, fakeClusterServiceBrokerClient, testController, sharedInformers := newTestController(t, fakeosb.FakeClientConfiguration{
2602+
UnbindReaction: &fakeosb.UnbindReaction{},
2603+
})
2604+
2605+
sharedInformers.ClusterServiceBrokers().Informer().GetStore().Add(getTestClusterServiceBroker())
2606+
sharedInformers.ClusterServiceClasses().Informer().GetStore().Add(getTestClusterServiceClass())
2607+
sharedInformers.ServiceInstances().Informer().GetStore().Add(getTestServiceInstanceWithRefs())
2608+
sharedInformers.ClusterServicePlans().Informer().GetStore().Add(getTestClusterServicePlan())
2609+
2610+
startTime := metav1.NewTime(time.Now().Add(-1 * time.Hour))
2611+
binding := &v1beta1.ServiceBinding{
2612+
ObjectMeta: metav1.ObjectMeta{
2613+
Name: testServiceBindingName,
2614+
Namespace: testNamespace,
2615+
DeletionTimestamp: &metav1.Time{},
2616+
Finalizers: []string{v1beta1.FinalizerServiceCatalog},
2617+
},
2618+
Spec: v1beta1.ServiceBindingSpec{
2619+
ServiceInstanceRef: v1beta1.LocalObjectReference{Name: testServiceInstanceName},
2620+
ExternalID: testServiceBindingGUID,
2621+
SecretName: testServiceBindingSecretName,
2622+
},
2623+
Status: v1beta1.ServiceBindingStatus{
2624+
CurrentOperation: v1beta1.ServiceBindingOperationBind,
2625+
OperationStartTime: &startTime,
2626+
OrphanMitigationInProgress: true,
2627+
},
2628+
}
2629+
2630+
fakeCatalogClient.AddReactor("get", "servicebindings", func(action clientgotesting.Action) (bool, runtime.Object, error) {
2631+
return true, binding, nil
2632+
})
2633+
2634+
timeOfReconciliation := metav1.Now()
2635+
2636+
err := testController.reconcileServiceBinding(binding)
2637+
if err != nil {
2638+
t.Fatalf("%v", err)
2639+
}
2640+
2641+
brokerActions := fakeClusterServiceBrokerClient.Actions()
2642+
assertNumberOfClusterServiceBrokerActions(t, brokerActions, 1)
2643+
assertUnbind(t, brokerActions[0], &osb.UnbindRequest{
2644+
BindingID: testServiceBindingGUID,
2645+
InstanceID: testServiceInstanceGUID,
2646+
ServiceID: testClusterServiceClassGUID,
2647+
PlanID: testClusterServicePlanGUID,
2648+
})
2649+
2650+
kubeActions := fakeKubeClient.Actions()
2651+
// The action should be deleting the secret
2652+
assertNumberOfActions(t, kubeActions, 1)
2653+
2654+
deleteAction := kubeActions[0].(clientgotesting.DeleteActionImpl)
2655+
if e, a := "delete", deleteAction.GetVerb(); e != a {
2656+
t.Fatalf("Unexpected verb on kubeActions[1]; expected %v, got %v", e, a)
2657+
}
2658+
2659+
if e, a := binding.Spec.SecretName, deleteAction.Name; e != a {
2660+
t.Fatalf("Unexpected name of secret: expected %v, got %v", e, a)
2661+
}
2662+
2663+
actions := fakeCatalogClient.Actions()
2664+
// The actions should be:
2665+
// 0. Updating the current operation
2666+
// 1. Updating the ready condition
2667+
assertNumberOfActions(t, actions, 2)
2668+
2669+
updatedServiceBinding := assertUpdateStatus(t, actions[0], binding).(*v1beta1.ServiceBinding)
2670+
assertServiceBindingOperationInProgress(t, updatedServiceBinding, v1beta1.ServiceBindingOperationUnbind, binding)
2671+
assertServiceBindingOrphanMitigationSet(t, updatedServiceBinding, false)
2672+
2673+
// Verify that the operation start time was reset to Now
2674+
if updatedServiceBinding.Status.OperationStartTime.Before(&timeOfReconciliation) {
2675+
t.Fatalf(
2676+
"OperationStartTime should not be before the time that the reconciliation started. OperationStartTime=%v. timeOfReconciliation=%v",
2677+
updatedServiceBinding.Status.OperationStartTime,
2678+
timeOfReconciliation,
2679+
)
2680+
}
2681+
2682+
updatedServiceBinding = assertUpdateStatus(t, actions[1], binding).(*v1beta1.ServiceBinding)
2683+
assertServiceBindingOperationSuccess(t, updatedServiceBinding, v1beta1.ServiceBindingOperationUnbind, binding)
2684+
assertServiceBindingOrphanMitigationSet(t, updatedServiceBinding, false)
2685+
2686+
events := getRecordedEvents(testController)
2687+
assertNumEvents(t, events, 1)
2688+
2689+
expectedEvent := corev1.EventTypeNormal + " " + successUnboundReason + " " + "This binding was deleted successfully"
2690+
if e, a := expectedEvent, events[0]; e != a {
2691+
t.Fatalf("Received unexpected event: %v", a)
24842692
}
24852693
}

‎pkg/controller/controller_instance_test.go

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4648,3 +4648,146 @@ func TestCheckClassAndPlanForDeletion(t *testing.T) {
46484648
}
46494649
}
46504650
}
4651+
4652+
// TestReconcileServiceInstanceDeleteDuringOngoingOperation tests deleting an
4653+
// instance that has an on-going operation.
4654+
func TestReconcileServiceInstanceDeleteDuringOngoingOperation(t *testing.T) {
4655+
fakeKubeClient, fakeCatalogClient, fakeClusterServiceBrokerClient, testController, sharedInformers := newTestController(t, fakeosb.FakeClientConfiguration{
4656+
DeprovisionReaction: &fakeosb.DeprovisionReaction{
4657+
Response: &osb.DeprovisionResponse{},
4658+
},
4659+
})
4660+
4661+
sharedInformers.ClusterServiceBrokers().Informer().GetStore().Add(getTestClusterServiceBroker())
4662+
sharedInformers.ClusterServiceClasses().Informer().GetStore().Add(getTestClusterServiceClass())
4663+
sharedInformers.ClusterServicePlans().Informer().GetStore().Add(getTestClusterServicePlan())
4664+
4665+
instance := getTestServiceInstanceWithRefs()
4666+
instance.ObjectMeta.DeletionTimestamp = &metav1.Time{}
4667+
instance.ObjectMeta.Finalizers = []string{v1beta1.FinalizerServiceCatalog}
4668+
instance.Status.CurrentOperation = v1beta1.ServiceInstanceOperationProvision
4669+
startTime := metav1.NewTime(time.Now().Add(-1 * time.Hour))
4670+
instance.Status.OperationStartTime = &startTime
4671+
4672+
fakeCatalogClient.AddReactor("get", "serviceinstances", func(action clientgotesting.Action) (bool, runtime.Object, error) {
4673+
return true, instance, nil
4674+
})
4675+
4676+
timeOfReconciliation := metav1.Now()
4677+
4678+
err := testController.reconcileServiceInstance(instance)
4679+
if err != nil {
4680+
t.Fatalf("This should not fail")
4681+
}
4682+
4683+
brokerActions := fakeClusterServiceBrokerClient.Actions()
4684+
assertNumberOfClusterServiceBrokerActions(t, brokerActions, 1)
4685+
assertDeprovision(t, brokerActions[0], &osb.DeprovisionRequest{
4686+
AcceptsIncomplete: true,
4687+
InstanceID: testServiceInstanceGUID,
4688+
ServiceID: testClusterServiceClassGUID,
4689+
PlanID: testClusterServicePlanGUID,
4690+
})
4691+
4692+
// Verify no core kube actions occurred
4693+
kubeActions := fakeKubeClient.Actions()
4694+
assertNumberOfActions(t, kubeActions, 0)
4695+
4696+
actions := fakeCatalogClient.Actions()
4697+
assertNumberOfActions(t, actions, 2)
4698+
4699+
updatedServiceInstance := assertUpdateStatus(t, actions[0], instance).(*v1beta1.ServiceInstance)
4700+
assertServiceInstanceOperationInProgress(t, updatedServiceInstance, v1beta1.ServiceInstanceOperationDeprovision, testClusterServicePlanName, instance)
4701+
4702+
// Verify that the operation start time was reset to Now
4703+
if updatedServiceInstance.Status.OperationStartTime.Before(&timeOfReconciliation) {
4704+
t.Fatalf(
4705+
"OperationStartTime should not be before the time that the reconciliation started. OperationStartTime=%v. timeOfReconciliation=%v",
4706+
updatedServiceInstance.Status.OperationStartTime,
4707+
timeOfReconciliation,
4708+
)
4709+
}
4710+
4711+
updatedServiceInstance = assertUpdateStatus(t, actions[1], instance).(*v1beta1.ServiceInstance)
4712+
assertServiceInstanceOperationSuccess(t, updatedServiceInstance, v1beta1.ServiceInstanceOperationDeprovision, testClusterServicePlanName, instance)
4713+
4714+
events := getRecordedEvents(testController)
4715+
assertNumEvents(t, events, 1)
4716+
4717+
expectedEvent := corev1.EventTypeNormal + " " + successDeprovisionReason + " " + "The instance was deprovisioned successfully"
4718+
if e, a := expectedEvent, events[0]; e != a {
4719+
t.Fatalf("Received unexpected event: %v\nExpected: %v", a, e)
4720+
}
4721+
}
4722+
4723+
// TestReconcileServiceInstanceDeleteDuringOrphanMitigation tests deleting an
4724+
// instance that is undergoing orphan mitigation.
4725+
func TestReconcileServiceInstanceDeleteWithOngoingOperation(t *testing.T) {
4726+
fakeKubeClient, fakeCatalogClient, fakeClusterServiceBrokerClient, testController, sharedInformers := newTestController(t, fakeosb.FakeClientConfiguration{
4727+
DeprovisionReaction: &fakeosb.DeprovisionReaction{
4728+
Response: &osb.DeprovisionResponse{},
4729+
},
4730+
})
4731+
4732+
sharedInformers.ClusterServiceBrokers().Informer().GetStore().Add(getTestClusterServiceBroker())
4733+
sharedInformers.ClusterServiceClasses().Informer().GetStore().Add(getTestClusterServiceClass())
4734+
sharedInformers.ClusterServicePlans().Informer().GetStore().Add(getTestClusterServicePlan())
4735+
4736+
instance := getTestServiceInstanceWithRefs()
4737+
instance.ObjectMeta.DeletionTimestamp = &metav1.Time{}
4738+
instance.ObjectMeta.Finalizers = []string{v1beta1.FinalizerServiceCatalog}
4739+
instance.Status.CurrentOperation = v1beta1.ServiceInstanceOperationProvision
4740+
startTime := metav1.NewTime(time.Now().Add(-1 * time.Hour))
4741+
instance.Status.OperationStartTime = &startTime
4742+
instance.Status.OrphanMitigationInProgress = true
4743+
4744+
fakeCatalogClient.AddReactor("get", "serviceinstances", func(action clientgotesting.Action) (bool, runtime.Object, error) {
4745+
return true, instance, nil
4746+
})
4747+
4748+
timeOfReconciliation := metav1.Now()
4749+
4750+
err := testController.reconcileServiceInstance(instance)
4751+
if err != nil {
4752+
t.Fatalf("This should not fail")
4753+
}
4754+
4755+
brokerActions := fakeClusterServiceBrokerClient.Actions()
4756+
assertNumberOfClusterServiceBrokerActions(t, brokerActions, 1)
4757+
assertDeprovision(t, brokerActions[0], &osb.DeprovisionRequest{
4758+
AcceptsIncomplete: true,
4759+
InstanceID: testServiceInstanceGUID,
4760+
ServiceID: testClusterServiceClassGUID,
4761+
PlanID: testClusterServicePlanGUID,
4762+
})
4763+
4764+
// Verify no core kube actions occurred
4765+
kubeActions := fakeKubeClient.Actions()
4766+
assertNumberOfActions(t, kubeActions, 0)
4767+
4768+
actions := fakeCatalogClient.Actions()
4769+
assertNumberOfActions(t, actions, 2)
4770+
4771+
updatedServiceInstance := assertUpdateStatus(t, actions[0], instance).(*v1beta1.ServiceInstance)
4772+
assertServiceInstanceOperationInProgress(t, updatedServiceInstance, v1beta1.ServiceInstanceOperationDeprovision, testClusterServicePlanName, instance)
4773+
4774+
// Verify that the operation start time was reset to Now
4775+
if updatedServiceInstance.Status.OperationStartTime.Before(&timeOfReconciliation) {
4776+
t.Fatalf(
4777+
"OperationStartTime should not be before the time that the reconciliation started. OperationStartTime=%v. timeOfReconciliation=%v",
4778+
updatedServiceInstance.Status.OperationStartTime,
4779+
timeOfReconciliation,
4780+
)
4781+
}
4782+
4783+
updatedServiceInstance = assertUpdateStatus(t, actions[1], instance).(*v1beta1.ServiceInstance)
4784+
assertServiceInstanceOperationSuccess(t, updatedServiceInstance, v1beta1.ServiceInstanceOperationDeprovision, testClusterServicePlanName, instance)
4785+
4786+
events := getRecordedEvents(testController)
4787+
assertNumEvents(t, events, 1)
4788+
4789+
expectedEvent := corev1.EventTypeNormal + " " + successDeprovisionReason + " " + "The instance was deprovisioned successfully"
4790+
if e, a := expectedEvent, events[0]; e != a {
4791+
t.Fatalf("Received unexpected event: %v\nExpected: %v", a, e)
4792+
}
4793+
}

0 commit comments

Comments
 (0)