diff --git a/test/e2e/crd_e2e_test.go b/test/e2e/crd_e2e_test.go index 2cbfb30be2..79091b6faf 100644 --- a/test/e2e/crd_e2e_test.go +++ b/test/e2e/crd_e2e_test.go @@ -3,6 +3,8 @@ package e2e import ( "context" "fmt" + operatorsv1 "github.com/operator-framework/api/pkg/operators/v1" + "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/bundle" "time" "github.com/blang/semver/v4" @@ -30,7 +32,18 @@ var _ = Describe("CRD Versions", func() { BeforeEach(func() { c = ctx.Ctx().KubeClient() crc = ctx.Ctx().OperatorClient() - generatedNamespace = SetupGeneratedTestNamespace(genName("crd-e2e-")) + namespace := genName("crd-e2e-") + og := operatorsv1.OperatorGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-operatorgroup", namespace), + Namespace: namespace, + Annotations: map[string]string{ + // Reduce the bundle unpack timeout to ensure error states are reached quickly + bundle.BundleUnpackTimeoutAnnotationKey: "5s", + }, + }, + } + generatedNamespace = SetupGeneratedTestNamespaceWithOperatorGroup(namespace, og) }) AfterEach(func() { @@ -113,184 +126,6 @@ var _ = Describe("CRD Versions", func() { GinkgoT().Logf("Install plan %s fetched with status %s", fetchedInstallPlan.GetName(), fetchedInstallPlan.Status.Phase) Expect(fetchedInstallPlan.Status.Phase).To(Equal(operatorsv1alpha1.InstallPlanPhaseComplete)) }) - - // issue:https://github.com/operator-framework/operator-lifecycle-manager/issues/2638 - It("[FLAKE] blocks a CRD upgrade that could cause data loss", func() { - By("checking the storage versions in the existing CRD status and the spec of the new CRD") - - mainPackageName := genName("nginx-update2-") - mainPackageStable := fmt.Sprintf("%s-stable", mainPackageName) - stableChannel := "stable" - - crdPlural := genName("ins-") - crdName := crdPlural + ".cluster.com" - oldCRD := apiextensionsv1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: crdName, - }, - Spec: apiextensionsv1.CustomResourceDefinitionSpec{ - Group: "cluster.com", - Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ - { - Name: "v1alpha2", - Served: true, - Storage: true, - Schema: &apiextensionsv1.CustomResourceValidation{ - OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ - Type: "object", - Description: "my crd schema", - }, - }, - }, - { - Name: "v2alpha1", - Served: true, - Storage: false, - Schema: &apiextensionsv1.CustomResourceValidation{ - OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ - Type: "object", - Description: "my crd schema", - }, - }, - }, - }, - Names: apiextensionsv1.CustomResourceDefinitionNames{ - Plural: crdPlural, - Singular: crdPlural, - Kind: crdPlural, - ListKind: "list" + crdPlural, - }, - Scope: apiextensionsv1.NamespaceScoped, - }, - } - - alphaChannel := "alpha" - mainPackageAlpha := fmt.Sprintf("%s-alpha", mainPackageName) - newCRD := apiextensionsv1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: crdName, - }, - Spec: apiextensionsv1.CustomResourceDefinitionSpec{ - Group: "cluster.com", - Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ - { - Name: "v1alpha3", - Served: true, - Storage: true, - Schema: &apiextensionsv1.CustomResourceValidation{ - OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ - Type: "object", - Description: "my crd schema", - }, - }, - }, - { - Name: "v2alpha2", - Served: true, - Storage: false, - Schema: &apiextensionsv1.CustomResourceValidation{ - OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ - Type: "object", - Description: "my crd schema", - }, - }, - }, - }, - Names: apiextensionsv1.CustomResourceDefinitionNames{ - Plural: crdPlural, - Singular: crdPlural, - Kind: crdPlural, - ListKind: "list" + crdPlural, - }, - Scope: apiextensionsv1.NamespaceScoped, - }, - } - - oldCSV := newCSV(mainPackageStable, generatedNamespace.GetName(), "", semver.MustParse("0.1.0"), []apiextensionsv1.CustomResourceDefinition{oldCRD}, nil, nil) - newCSV := newCSV(mainPackageAlpha, generatedNamespace.GetName(), mainPackageStable, semver.MustParse("0.1.1"), []apiextensionsv1.CustomResourceDefinition{newCRD}, nil, nil) - mainCatalogName := genName("mock-ocs-main-update2-") - mainManifests := []registry.PackageManifest{ - { - PackageName: mainPackageName, - Channels: []registry.PackageChannel{ - {Name: stableChannel, CurrentCSVName: mainPackageStable}, - {Name: alphaChannel, CurrentCSVName: mainPackageAlpha}, - }, - DefaultChannelName: stableChannel, - }, - } - - By("Create the catalog sources") - _, cleanupMainCatalogSource := createInternalCatalogSource(c, crc, mainCatalogName, generatedNamespace.GetName(), mainManifests, []apiextensionsv1.CustomResourceDefinition{oldCRD, newCRD}, []operatorsv1alpha1.ClusterServiceVersion{oldCSV, newCSV}) - defer cleanupMainCatalogSource() - defer func() { - _ = crc.OperatorsV1alpha1().ClusterServiceVersions(generatedNamespace.GetName()).Delete(context.TODO(), oldCSV.GetName(), metav1.DeleteOptions{}) - _ = crc.OperatorsV1alpha1().ClusterServiceVersions(generatedNamespace.GetName()).Delete(context.TODO(), newCSV.GetName(), metav1.DeleteOptions{}) - _ = c.ApiextensionsInterface().ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), oldCRD.GetName(), metav1.DeleteOptions{}) - _ = c.ApiextensionsInterface().ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), newCRD.GetName(), metav1.DeleteOptions{}) - }() - - By("Attempt to get the catalog source before creating install plan") - _, err := fetchCatalogSourceOnStatus(crc, mainCatalogName, generatedNamespace.GetName(), catalogSourceRegistryPodSynced()) - Expect(err).ToNot(HaveOccurred()) - - subscriptionName := genName("sub-nginx-update2-") - subscriptionCleanup := createSubscriptionForCatalog(crc, generatedNamespace.GetName(), subscriptionName, mainCatalogName, mainPackageName, stableChannel, "", operatorsv1alpha1.ApprovalAutomatic) - defer subscriptionCleanup() - - subscription, err := fetchSubscription(crc, generatedNamespace.GetName(), subscriptionName, subscriptionHasInstallPlanChecker()) - Expect(err).ToNot(HaveOccurred()) - Expect(subscription).ToNot(BeNil()) - Expect(subscription.Status.InstallPlanRef).ToNot(Equal(nil)) - Expect(oldCSV.GetName()).To(Equal(subscription.Status.CurrentCSV)) - - installPlanName := subscription.Status.InstallPlanRef.Name - - By("Wait for InstallPlan to be status: Complete before checking resource presence") - fetchedInstallPlan, err := fetchInstallPlan(GinkgoT(), crc, installPlanName, generatedNamespace.GetName(), buildInstallPlanPhaseCheckFunc(operatorsv1alpha1.InstallPlanPhaseComplete)) - Expect(err).ToNot(HaveOccurred()) - GinkgoT().Logf("Install plan %s fetched with status %s", fetchedInstallPlan.GetName(), fetchedInstallPlan.Status.Phase) - Expect(fetchedInstallPlan.Status.Phase).To(Equal(operatorsv1alpha1.InstallPlanPhaseComplete)) - - By("old CRD has been installed onto the cluster - now upgrade the subscription to point to the channel with the new CRD") - By("installing the new CSV should fail with a warning about data loss, since a storage version is missing in the new CRD") - By("use server-side apply to apply the update to the subscription point to the alpha channel") - Eventually(Apply(subscription, func(s *operatorsv1alpha1.Subscription) error { - s.Spec.Channel = alphaChannel - return nil - })).Should(Succeed()) - ctx.Ctx().Logf("updated subscription to point to alpha channel") - - checker := subscriptionStateAtLatestChecker() - subscriptionAtLatestWithDifferentInstallPlan := func(v *operatorsv1alpha1.Subscription) bool { - return checker(v) && v.Status.InstallPlanRef != nil && v.Status.InstallPlanRef.Name != fetchedInstallPlan.Name - } - - By("fetch new subscription") - s, err := fetchSubscription(crc, generatedNamespace.GetName(), subscriptionName, subscriptionAtLatestWithDifferentInstallPlan) - Expect(err).ToNot(HaveOccurred()) - Expect(s).ToNot(BeNil()) - Expect(s.Status.InstallPlanRef).ToNot(Equal(nil)) - - By("Check the error on the installplan - should be related to data loss and the CRD upgrade missing a stored version") - Eventually(func() (*operatorsv1alpha1.InstallPlan, error) { - return crc.OperatorsV1alpha1().InstallPlans(generatedNamespace.GetName()).Get(context.TODO(), s.Status.InstallPlanRef.Name, metav1.GetOptions{}) - }).Should(And( - WithTransform( - func(v *operatorsv1alpha1.InstallPlan) operatorsv1alpha1.InstallPlanPhase { - return v.Status.Phase - }, - Equal(operatorsv1alpha1.InstallPlanPhaseFailed), - ), - WithTransform( - func(v *operatorsv1alpha1.InstallPlan) string { - return v.Status.Conditions[len(v.Status.Conditions)-1].Message - }, - ContainSubstring("risk of data loss"), - ), - )) - }) - It("allows a CRD upgrade that doesn't cause data loss", func() { By(`Create a CRD on cluster with v1alpha1 (storage)`) By(`Update that CRD with v1alpha2 (storage), v1alpha1 (served)`) @@ -461,7 +296,7 @@ var _ = Describe("CRD Versions", func() { err = crc.OperatorsV1alpha1().Subscriptions(generatedNamespace.GetName()).Delete(context.TODO(), subscription.GetName(), metav1.DeleteOptions{}) Expect(err).ToNot(HaveOccurred(), "error deleting old subscription") By("remove old csv") - crc.OperatorsV1alpha1().ClusterServiceVersions(generatedNamespace.GetName()).Delete(context.TODO(), mainPackageStable, metav1.DeleteOptions{}) + err = crc.OperatorsV1alpha1().ClusterServiceVersions(generatedNamespace.GetName()).Delete(context.TODO(), mainPackageStable, metav1.DeleteOptions{}) Expect(err).ToNot(HaveOccurred(), "error deleting old subscription") By("recreate subscription") @@ -493,4 +328,181 @@ var _ = Describe("CRD Versions", func() { }).Should(BeTrue()) GinkgoT().Log("manually reconciled potentially unsafe CRD upgrade") }) + + It("blocks a CRD upgrade that could cause data loss", func() { + By("checking the storage versions in the existing CRD status and the spec of the new CRD") + + mainPackageName := genName("nginx-update2-") + mainPackageStable := fmt.Sprintf("%s-stable", mainPackageName) + stableChannel := "stable" + + crdPlural := genName("ins-") + crdName := crdPlural + ".cluster.com" + oldCRD := apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: crdName, + }, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "cluster.com", + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1alpha2", + Served: true, + Storage: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", + Description: "my crd schema", + }, + }, + }, + { + Name: "v2alpha1", + Served: true, + Storage: false, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", + Description: "my crd schema", + }, + }, + }, + }, + Names: apiextensionsv1.CustomResourceDefinitionNames{ + Plural: crdPlural, + Singular: crdPlural, + Kind: crdPlural, + ListKind: "list" + crdPlural, + }, + Scope: apiextensionsv1.NamespaceScoped, + }, + } + + alphaChannel := "alpha" + mainPackageAlpha := fmt.Sprintf("%s-alpha", mainPackageName) + newCRD := apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: crdName, + }, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "cluster.com", + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1alpha3", + Served: true, + Storage: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", + Description: "my crd schema", + }, + }, + }, + { + Name: "v2alpha2", + Served: true, + Storage: false, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", + Description: "my crd schema", + }, + }, + }, + }, + Names: apiextensionsv1.CustomResourceDefinitionNames{ + Plural: crdPlural, + Singular: crdPlural, + Kind: crdPlural, + ListKind: "list" + crdPlural, + }, + Scope: apiextensionsv1.NamespaceScoped, + }, + } + + oldCSV := newCSV(mainPackageStable, generatedNamespace.GetName(), "", semver.MustParse("0.1.0"), []apiextensionsv1.CustomResourceDefinition{oldCRD}, nil, nil) + newCSV := newCSV(mainPackageAlpha, generatedNamespace.GetName(), mainPackageStable, semver.MustParse("0.1.1"), []apiextensionsv1.CustomResourceDefinition{newCRD}, nil, nil) + mainCatalogName := genName("mock-ocs-main-update2-") + mainManifests := []registry.PackageManifest{ + { + PackageName: mainPackageName, + Channels: []registry.PackageChannel{ + {Name: stableChannel, CurrentCSVName: mainPackageStable}, + {Name: alphaChannel, CurrentCSVName: mainPackageAlpha}, + }, + DefaultChannelName: stableChannel, + }, + } + + By("Create the catalog sources") + _, cleanupMainCatalogSource := createInternalCatalogSource(c, crc, mainCatalogName, generatedNamespace.GetName(), mainManifests, []apiextensionsv1.CustomResourceDefinition{oldCRD, newCRD}, []operatorsv1alpha1.ClusterServiceVersion{oldCSV, newCSV}) + defer cleanupMainCatalogSource() + defer func() { + _ = crc.OperatorsV1alpha1().ClusterServiceVersions(generatedNamespace.GetName()).Delete(context.TODO(), oldCSV.GetName(), metav1.DeleteOptions{}) + _ = crc.OperatorsV1alpha1().ClusterServiceVersions(generatedNamespace.GetName()).Delete(context.TODO(), newCSV.GetName(), metav1.DeleteOptions{}) + _ = c.ApiextensionsInterface().ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), oldCRD.GetName(), metav1.DeleteOptions{}) + _ = c.ApiextensionsInterface().ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), newCRD.GetName(), metav1.DeleteOptions{}) + }() + + By("Attempt to get the catalog source before creating install plan") + _, err := fetchCatalogSourceOnStatus(crc, mainCatalogName, generatedNamespace.GetName(), catalogSourceRegistryPodSynced()) + Expect(err).ToNot(HaveOccurred()) + + subscriptionName := genName("sub-nginx-update2-") + subscriptionCleanup := createSubscriptionForCatalog(crc, generatedNamespace.GetName(), subscriptionName, mainCatalogName, mainPackageName, stableChannel, "", operatorsv1alpha1.ApprovalAutomatic) + defer subscriptionCleanup() + + subscription, err := fetchSubscription(crc, generatedNamespace.GetName(), subscriptionName, subscriptionHasInstallPlanChecker()) + Expect(err).ToNot(HaveOccurred()) + Expect(subscription).ToNot(BeNil()) + Expect(subscription.Status.InstallPlanRef).ToNot(Equal(nil)) + Expect(oldCSV.GetName()).To(Equal(subscription.Status.CurrentCSV)) + + installPlanName := subscription.Status.InstallPlanRef.Name + + By("Wait for InstallPlan to be status: Complete before checking resource presence") + fetchedInstallPlan, err := fetchInstallPlan(GinkgoT(), crc, installPlanName, generatedNamespace.GetName(), buildInstallPlanPhaseCheckFunc(operatorsv1alpha1.InstallPlanPhaseComplete)) + Expect(err).ToNot(HaveOccurred()) + GinkgoT().Logf("Install plan %s fetched with status %s", fetchedInstallPlan.GetName(), fetchedInstallPlan.Status.Phase) + Expect(fetchedInstallPlan.Status.Phase).To(Equal(operatorsv1alpha1.InstallPlanPhaseComplete)) + + By("old CRD has been installed onto the cluster - now upgrade the subscription to point to the channel with the new CRD") + By("installing the new CSV should fail with a warning about data loss, since a storage version is missing in the new CRD") + By("use server-side apply to apply the update to the subscription point to the alpha channel") + Eventually(Apply(subscription, func(s *operatorsv1alpha1.Subscription) error { + s.Spec.Channel = alphaChannel + return nil + })).Should(Succeed()) + ctx.Ctx().Logf("updated subscription to point to alpha channel") + + checker := subscriptionStateAtLatestChecker() + subscriptionAtLatestWithDifferentInstallPlan := func(v *operatorsv1alpha1.Subscription) bool { + return checker(v) && v.Status.InstallPlanRef != nil && v.Status.InstallPlanRef.Name != fetchedInstallPlan.Name + } + + By("fetch new subscription") + s, err := fetchSubscription(crc, generatedNamespace.GetName(), subscriptionName, subscriptionAtLatestWithDifferentInstallPlan) + Expect(err).ToNot(HaveOccurred()) + Expect(s).ToNot(BeNil()) + Expect(s.Status.InstallPlanRef).ToNot(Equal(nil)) + + By("Check the error on the installplan - should be related to data loss and the CRD upgrade missing a stored version") + Eventually(func() (*operatorsv1alpha1.InstallPlan, error) { + return crc.OperatorsV1alpha1().InstallPlans(generatedNamespace.GetName()).Get(context.TODO(), s.Status.InstallPlanRef.Name, metav1.GetOptions{}) + // the install plan retry time out is 60 seconds, so we should expect the install plan to be failed within 2 minutes + }).Within(2 * time.Minute).Should(And( + WithTransform( + func(v *operatorsv1alpha1.InstallPlan) operatorsv1alpha1.InstallPlanPhase { + return v.Status.Phase + }, + Equal(operatorsv1alpha1.InstallPlanPhaseFailed), + ), + WithTransform( + func(v *operatorsv1alpha1.InstallPlan) string { + return v.Status.Conditions[len(v.Status.Conditions)-1].Message + }, + ContainSubstring("risk of data loss"), + ), + )) + }) })