Skip to content

Commit 30f6e83

Browse files
sadasusadasu
committed
MAO to stop managing metal3 deployments if CBO is available
In an upgrade scenario, check if CBO is available to take ownership of the metal3 deployment. In that case, back off and let CBO manage it. In a downgrade/rollback scenario where the metal3 deployment is annotated as being owned by cluster-baremetal-operator, but the baremetal operator has been removed, take back control of the metal3 deployment.
1 parent 79ac5f8 commit 30f6e83

File tree

4 files changed

+213
-1
lines changed

4 files changed

+213
-1
lines changed

‎pkg/operator/baremetal_pod.go

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,19 @@ import (
99
"golang.org/x/crypto/bcrypt"
1010

1111
"github.com/golang/glog"
12+
osclientset "github.com/openshift/client-go/config/clientset/versioned"
1213
appsv1 "k8s.io/api/apps/v1"
1314
corev1 "k8s.io/api/core/v1"
1415
apierrors "k8s.io/apimachinery/pkg/api/errors"
1516
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
17+
appsclientv1 "k8s.io/client-go/kubernetes/typed/apps/v1"
1618
coreclientv1 "k8s.io/client-go/kubernetes/typed/core/v1"
1719
"k8s.io/utils/pointer"
1820
)
1921

2022
const (
2123
baremetalConfigmap = "metal3-config"
24+
baremetalDeploymentName = "metal3"
2225
baremetalSharedVolume = "metal3-shared"
2326
baremetalSecretName = "metal3-mariadb-password"
2427
baremetalSecretKey = "password"
@@ -235,13 +238,40 @@ func createMetal3PasswordSecrets(client coreclientv1.SecretsGetter, config *Oper
235238
return nil
236239
}
237240

241+
// Return false on error or if "baremetal.openshift.io/owned" annotation set
242+
func checkMetal3DeploymentMAOOwned(client appsclientv1.DeploymentsGetter, config *OperatorConfig) (bool, error) {
243+
existing, err := client.Deployments(config.TargetNamespace).Get(context.Background(), "metal3", metav1.GetOptions{})
244+
if err != nil {
245+
if apierrors.IsNotFound(err) {
246+
return true, nil
247+
}
248+
return false, err
249+
}
250+
if _, exists := existing.ObjectMeta.Annotations[cboOwnedAnnotation]; exists {
251+
return false, nil
252+
}
253+
return true, nil
254+
}
255+
256+
// Return true if the baremetal clusteroperator exists
257+
func checkForBaremetalClusterOperator(osClient osclientset.Interface) (bool, error) {
258+
_, err := osClient.ConfigV1().ClusterOperators().Get(context.Background(), cboClusterOperatorName, metav1.GetOptions{})
259+
if err != nil {
260+
if apierrors.IsNotFound(err) {
261+
return false, nil
262+
}
263+
return false, err
264+
}
265+
return true, nil
266+
}
267+
238268
func newMetal3Deployment(config *OperatorConfig, baremetalProvisioningConfig BaremetalProvisioningConfig) *appsv1.Deployment {
239269
replicas := int32(1)
240270
template := newMetal3PodTemplateSpec(config, baremetalProvisioningConfig)
241271

242272
return &appsv1.Deployment{
243273
ObjectMeta: metav1.ObjectMeta{
244-
Name: "metal3",
274+
Name: baremetalDeploymentName,
245275
Namespace: config.TargetNamespace,
246276
Annotations: map[string]string{
247277
maoOwnedAnnotation: "",

‎pkg/operator/baremetal_test.go

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,11 @@ import (
44
"testing"
55

66
. "github.com/onsi/gomega"
7+
osconfigv1 "github.com/openshift/api/config/v1"
8+
fakeos "github.com/openshift/client-go/config/clientset/versioned/fake"
79
"golang.org/x/net/context"
10+
appsv1 "k8s.io/api/apps/v1"
11+
apierrors "k8s.io/apimachinery/pkg/api/errors"
812
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
913
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
1014
"k8s.io/apimachinery/pkg/runtime"
@@ -341,3 +345,160 @@ func TestBaremetalProvisionigConfig(t *testing.T) {
341345
})
342346
}
343347
}
348+
349+
func TestCheckMetal3DeploymentOwned(t *testing.T) {
350+
kubeClient := fakekube.NewSimpleClientset(nil...)
351+
operatorConfig := newOperatorWithBaremetalConfig()
352+
client := kubeClient.AppsV1()
353+
354+
testCases := []struct {
355+
testCase string
356+
deployment *appsv1.Deployment
357+
expected bool
358+
expectedError bool
359+
}{
360+
{
361+
testCase: "Only maoOwnedAnnotation",
362+
deployment: &appsv1.Deployment{
363+
TypeMeta: metav1.TypeMeta{
364+
Kind: "Deployment",
365+
APIVersion: "apps/v1",
366+
},
367+
ObjectMeta: metav1.ObjectMeta{
368+
Name: "metal3",
369+
Annotations: map[string]string{
370+
maoOwnedAnnotation: "",
371+
},
372+
},
373+
},
374+
expected: true,
375+
},
376+
{
377+
testCase: "Only cboOwnedAnnotation",
378+
deployment: &appsv1.Deployment{
379+
TypeMeta: metav1.TypeMeta{
380+
Kind: "Deployment",
381+
APIVersion: "apps/v1",
382+
},
383+
ObjectMeta: metav1.ObjectMeta{
384+
Name: "metal3",
385+
Annotations: map[string]string{
386+
cboOwnedAnnotation: "",
387+
},
388+
},
389+
},
390+
expected: false,
391+
},
392+
{
393+
testCase: "Both cboOwnedAnnotation and maoOwnedAnnotation",
394+
deployment: &appsv1.Deployment{
395+
TypeMeta: metav1.TypeMeta{
396+
Kind: "Deployment",
397+
APIVersion: "apps/v1",
398+
},
399+
ObjectMeta: metav1.ObjectMeta{
400+
Name: "metal3",
401+
Annotations: map[string]string{
402+
cboOwnedAnnotation: "",
403+
maoOwnedAnnotation: "",
404+
},
405+
},
406+
},
407+
expected: false,
408+
},
409+
{
410+
testCase: "No cboOwnedAnnotation or maoOwnedAnnotation",
411+
deployment: &appsv1.Deployment{
412+
TypeMeta: metav1.TypeMeta{
413+
Kind: "Deployment",
414+
APIVersion: "apps/v1",
415+
},
416+
ObjectMeta: metav1.ObjectMeta{
417+
Name: "metal3",
418+
Annotations: map[string]string{},
419+
},
420+
},
421+
expected: true,
422+
},
423+
}
424+
for _, tc := range testCases {
425+
t.Run(string(tc.testCase), func(t *testing.T) {
426+
427+
_, err := client.Deployments("test-namespace").Create(context.Background(), tc.deployment, metav1.CreateOptions{})
428+
if err != nil {
429+
t.Fatalf("Could not create metal3 test deployment.\n")
430+
}
431+
maoOwned, err := checkMetal3DeploymentMAOOwned(client, operatorConfig)
432+
if maoOwned != tc.expected {
433+
t.Errorf("Expected: %v, got: %v", tc.expected, maoOwned)
434+
}
435+
if tc.expectedError != (err != nil) {
436+
t.Errorf("ExpectedError: %v, got: %v", tc.expectedError, err)
437+
}
438+
err = client.Deployments("test-namespace").Delete(context.Background(), "metal3", metav1.DeleteOptions{})
439+
if err != nil {
440+
t.Errorf("Could not delete metal3 test deployment.\n")
441+
}
442+
})
443+
}
444+
445+
}
446+
447+
func TestCheckForBaremetalClusterOperator(t *testing.T) {
448+
testCases := []struct {
449+
testCase string
450+
clusterOperator *osconfigv1.ClusterOperator
451+
expected bool
452+
expectedError bool
453+
}{
454+
{
455+
testCase: cboClusterOperatorName,
456+
clusterOperator: &osconfigv1.ClusterOperator{
457+
TypeMeta: metav1.TypeMeta{
458+
Kind: "ClusterOperator",
459+
APIVersion: "config.openshift.io/v1",
460+
},
461+
ObjectMeta: metav1.ObjectMeta{
462+
Name: cboClusterOperatorName,
463+
},
464+
Status: osconfigv1.ClusterOperatorStatus{
465+
RelatedObjects: []osconfigv1.ObjectReference{
466+
{
467+
Group: "",
468+
Resource: "namespaces",
469+
Name: "openshift-machine-api",
470+
},
471+
},
472+
},
473+
},
474+
expected: true,
475+
},
476+
{
477+
testCase: "invalidCO",
478+
clusterOperator: &osconfigv1.ClusterOperator{
479+
ObjectMeta: metav1.ObjectMeta{
480+
Name: "invalidCO",
481+
},
482+
},
483+
expected: false,
484+
},
485+
}
486+
for _, tc := range testCases {
487+
t.Run(string(tc.testCase), func(t *testing.T) {
488+
var osClient *fakeos.Clientset
489+
osClient = fakeos.NewSimpleClientset(tc.clusterOperator)
490+
_, err := osClient.ConfigV1().ClusterOperators().Create(context.Background(), tc.clusterOperator, metav1.CreateOptions{})
491+
if err != nil && !apierrors.IsAlreadyExists(err) {
492+
t.Fatalf("Unable to create ClusterOperator for test: %v", err)
493+
}
494+
exists, err := checkForBaremetalClusterOperator(osClient)
495+
if exists != tc.expected {
496+
t.Errorf("Expected: %v, got: %v", tc.expected, exists)
497+
}
498+
if tc.expectedError != (err != nil) {
499+
t.Errorf("ExpectedError: %v, got: %v", tc.expectedError, err)
500+
}
501+
err = osClient.ConfigV1().ClusterOperators().Delete(context.Background(), tc.testCase, metav1.DeleteOptions{})
502+
})
503+
}
504+
}

‎pkg/operator/operator.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ const (
3535
// 5ms, 10ms, 20ms, 40ms, 80ms, 160ms, 320ms, 640ms, 1.3s, 2.6s, 5.1s, 10.2s, 20.4s, 41s, 82s
3636
maxRetries = 15
3737
maoOwnedAnnotation = "machine.openshift.io/owned"
38+
// Indicates that the metal3 deployment is being managed by cluster-baremetal-operator
39+
cboOwnedAnnotation = "baremetal.openshift.io/owned"
40+
// The name of the clusteroperator for cluster-baremetal-operator
41+
cboClusterOperatorName = "baremetal"
3842
)
3943

4044
// Operator defines machine api operator.

‎pkg/operator/sync.go

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,23 @@ func (optr *Operator) syncMutatingWebhook() error {
193193
}
194194

195195
func (optr *Operator) syncBaremetalControllers(config *OperatorConfig) error {
196+
// Stand down if cluster-bare-metal operator has claimed the metal3 deployment
197+
// and if the baremetal clusteroperator exists
198+
maoOwned, err := checkMetal3DeploymentMAOOwned(optr.kubeClient.AppsV1(), config)
199+
if err != nil {
200+
return err
201+
}
202+
if !maoOwned {
203+
cboExists, err := checkForBaremetalClusterOperator(optr.osClient)
204+
if err != nil {
205+
return err
206+
}
207+
if cboExists {
208+
glog.Infof("cluster-baremetal-operator is running and managing the Metal3 deployment, standing down.")
209+
return nil
210+
}
211+
}
212+
196213
// Try to get baremetal provisioning config from a CR
197214
baremetalProvisioningConfig, err := getBaremetalProvisioningConfig(optr.dynamicClient, baremetalProvisioningCR)
198215
if err != nil {

0 commit comments

Comments
 (0)