MAO to stop managing metal3 deployments if CBO is available

In an upgrade scenario, check if CBO is available to take ownership
of the metal3 deployment. In that case, back off and let CBO manage it.

In a downgrade/rollback scenario where the metal3 deployment is
annotated as being owned by cluster-baremetal-operator, but the
baremetal operator has been removed, take back control of the metal3
deployment.

Author: sadasu

pkg/operator/baremetal_pod.go

@@ -9,15 +9,18 @@ import (
 	"golang.org/x/crypto/bcrypt"
 
 	"github.com/golang/glog"
+	osclientset "github.com/openshift/client-go/config/clientset/versioned"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	appsclientv1 "k8s.io/client-go/kubernetes/typed/apps/v1"
 	coreclientv1 "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/utils/pointer"
 )
 
 const (
+	baremetalDeploymentName    = "metal3"
 	baremetalSharedVolume      = "metal3-shared"
 	baremetalSecretName        = "metal3-mariadb-password"
 	baremetalSecretKey         = "password"
@@ -234,13 +237,40 @@ func createMetal3PasswordSecrets(client coreclientv1.SecretsGetter, config *Oper
 	return nil
 }
 
+// Return false on error or if "baremetal.openshift.io/owned" annotation set
+func checkMetal3DeploymentMAOOwned(client appsclientv1.DeploymentsGetter, config *OperatorConfig) (bool, error) {
+	existing, err := client.Deployments(config.TargetNamespace).Get(context.Background(), baremetalDeploymentName, metav1.GetOptions{})
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return true, nil
+		}
+		return false, err
+	}
+	if _, exists := existing.ObjectMeta.Annotations[cboOwnedAnnotation]; exists {
+		return false, nil
+	}
+	return true, nil
+}
+
+// Return true if the baremetal clusteroperator exists
+func checkForBaremetalClusterOperator(osClient osclientset.Interface) (bool, error) {
+	_, err := osClient.ConfigV1().ClusterOperators().Get(context.Background(), cboClusterOperatorName, metav1.GetOptions{})
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	return true, nil
+}
+
 func newMetal3Deployment(config *OperatorConfig, baremetalProvisioningConfig BaremetalProvisioningConfig) *appsv1.Deployment {
 	replicas := int32(1)
 	template := newMetal3PodTemplateSpec(config, baremetalProvisioningConfig)
 
 	return &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "metal3",
+			Name:      baremetalDeploymentName,
 			Namespace: config.TargetNamespace,
 			Annotations: map[string]string{
 				maoOwnedAnnotation: "",

pkg/operator/baremetal_test.go

@@ -4,7 +4,10 @@ import (
 	"testing"
 
 	. "github.com/onsi/gomega"
+	osconfigv1 "github.com/openshift/api/config/v1"
+	fakeos "github.com/openshift/client-go/config/clientset/versioned/fake"
 	"golang.org/x/net/context"
+	appsv1 "k8s.io/api/apps/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -375,7 +378,163 @@ func TestSyncBaremetalControllers(t *testing.T) {
 
 			err := optr.syncBaremetalControllers(operatorConfig, tc.configCRName)
 			g.Expect(err).To(Equal(tc.expectedError))
+		})
+	}
+}
+
+func TestCheckMetal3DeploymentOwned(t *testing.T) {
+	kubeClient := fakekube.NewSimpleClientset(nil...)
+	operatorConfig := newOperatorWithBaremetalConfig()
+	client := kubeClient.AppsV1()
 
+	testCases := []struct {
+		testCase      string
+		deployment    *appsv1.Deployment
+		expected      bool
+		expectedError bool
+	}{
+		{
+			testCase: "Only maoOwnedAnnotation",
+			deployment: &appsv1.Deployment{
+				TypeMeta: metav1.TypeMeta{
+					Kind:       "Deployment",
+					APIVersion: "apps/v1",
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Name: baremetalDeploymentName,
+					Annotations: map[string]string{
+						maoOwnedAnnotation: "",
+					},
+				},
+			},
+			expected: true,
+		},
+		{
+			testCase: "Only cboOwnedAnnotation",
+			deployment: &appsv1.Deployment{
+				TypeMeta: metav1.TypeMeta{
+					Kind:       "Deployment",
+					APIVersion: "apps/v1",
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Name: baremetalDeploymentName,
+					Annotations: map[string]string{
+						cboOwnedAnnotation: "",
+					},
+				},
+			},
+			expected: false,
+		},
+		{
+			testCase: "Both cboOwnedAnnotation and maoOwnedAnnotation",
+			deployment: &appsv1.Deployment{
+				TypeMeta: metav1.TypeMeta{
+					Kind:       "Deployment",
+					APIVersion: "apps/v1",
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Name: baremetalDeploymentName,
+					Annotations: map[string]string{
+						cboOwnedAnnotation: "",
+						maoOwnedAnnotation: "",
+					},
+				},
+			},
+			expected: false,
+		},
+		{
+			testCase: "No cboOwnedAnnotation or maoOwnedAnnotation",
+			deployment: &appsv1.Deployment{
+				TypeMeta: metav1.TypeMeta{
+					Kind:       "Deployment",
+					APIVersion: "apps/v1",
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Name:        baremetalDeploymentName,
+					Annotations: map[string]string{},
+				},
+			},
+			expected: true,
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(string(tc.testCase), func(t *testing.T) {
+
+			_, err := client.Deployments("test-namespace").Create(context.Background(), tc.deployment, metav1.CreateOptions{})
+			if err != nil {
+				t.Fatalf("Could not create metal3 test deployment.\n")
+			}
+			maoOwned, err := checkMetal3DeploymentMAOOwned(client, operatorConfig)
+			if maoOwned != tc.expected {
+				t.Errorf("Expected: %v, got: %v", tc.expected, maoOwned)
+			}
+			if tc.expectedError != (err != nil) {
+				t.Errorf("ExpectedError: %v, got: %v", tc.expectedError, err)
+			}
+			err = client.Deployments("test-namespace").Delete(context.Background(), baremetalDeploymentName, metav1.DeleteOptions{})
+			if err != nil {
+				t.Errorf("Could not delete metal3 test deployment.\n")
+			}
+		})
+	}
+
+}
+
+func TestCheckForBaremetalClusterOperator(t *testing.T) {
+	testCases := []struct {
+		testCase        string
+		clusterOperator *osconfigv1.ClusterOperator
+		expected        bool
+		expectedError   bool
+	}{
+		{
+			testCase: cboClusterOperatorName,
+			clusterOperator: &osconfigv1.ClusterOperator{
+				TypeMeta: metav1.TypeMeta{
+					Kind:       "ClusterOperator",
+					APIVersion: "config.openshift.io/v1",
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Name: cboClusterOperatorName,
+				},
+				Status: osconfigv1.ClusterOperatorStatus{
+					RelatedObjects: []osconfigv1.ObjectReference{
+						{
+							Group:    "",
+							Resource: "namespaces",
+							Name:     "openshift-machine-api",
+						},
+					},
+				},
+			},
+			expected: true,
+		},
+		{
+			testCase: "invalidCO",
+			clusterOperator: &osconfigv1.ClusterOperator{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "invalidCO",
+				},
+			},
+			expected: false,
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(string(tc.testCase), func(t *testing.T) {
+			var osClient *fakeos.Clientset
+			osClient = fakeos.NewSimpleClientset(tc.clusterOperator)
+			_, err := osClient.ConfigV1().ClusterOperators().Create(context.Background(), tc.clusterOperator, metav1.CreateOptions{})
+			if err != nil && !apierrors.IsAlreadyExists(err) {
+				t.Fatalf("Unable to create ClusterOperator for test: %v", err)
+			}
+			exists, err := checkForBaremetalClusterOperator(osClient)
+			if exists != tc.expected {
+				t.Errorf("Expected: %v, got: %v", tc.expected, exists)
+			}
+			if tc.expectedError != (err != nil) {
+				t.Errorf("ExpectedError: %v, got: %v", tc.expectedError, err)
+			}
+			err = osClient.ConfigV1().ClusterOperators().Delete(context.Background(), tc.testCase, metav1.DeleteOptions{})
 		})
 	}
 }

pkg/operator/operator.go

@@ -35,6 +35,10 @@ const (
 	// 5ms, 10ms, 20ms, 40ms, 80ms, 160ms, 320ms, 640ms, 1.3s, 2.6s, 5.1s, 10.2s, 20.4s, 41s, 82s
 	maxRetries         = 15
 	maoOwnedAnnotation = "machine.openshift.io/owned"
+	// Indicates that the metal3 deployment is being managed by cluster-baremetal-operator
+	cboOwnedAnnotation = "baremetal.openshift.io/owned"
+	// The name of the clusteroperator for cluster-baremetal-operator
+	cboClusterOperatorName = "baremetal"
 )
 
 // Operator defines machine api operator.

pkg/operator/sync.go

@@ -198,6 +198,23 @@ func (optr *Operator) syncMutatingWebhook() error {
 }
 
 func (optr *Operator) syncBaremetalControllers(config *OperatorConfig, configName string) error {
+	// Stand down if cluster-bare-metal operator has claimed the metal3 deployment
+	// and if the baremetal clusteroperator exists
+	maoOwned, err := checkMetal3DeploymentMAOOwned(optr.kubeClient.AppsV1(), config)
+	if err != nil {
+		return err
+	}
+	if !maoOwned {
+		cboExists, err := checkForBaremetalClusterOperator(optr.osClient)
+		if err != nil {
+			return err
+		}
+		if cboExists {
+			glog.Infof("cluster-baremetal-operator is running and managing the Metal3 deployment, standing down.")
+			return nil
+		}
+	}
+
 	// Try to get baremetal provisioning config from a CR
 	baremetalProvisioningConfig, err := getBaremetalProvisioningConfig(optr.dynamicClient, configName)
 	if err == nil && baremetalProvisioningConfig == nil {