Refactor karpenter deployment to CPOv2

This commit refactors the karpenter deployment and assets which are managed by the karpenter
operator to CPOv2. This removes the ability to pass in the karpenter aws image as an argument
to the karpenter-operator deployment, since we can directly get the karpenter aws image from
the payload image itself from within the karpenter-operator releaseImage lookup code. This
commit also removes the deprecated karpenter code.

Signed-off-by: Max Cao <[email protected]>

Author: maxcao13

control-plane-operator/controllers/hostedcontrolplane/v2/assets/karpenter/deployment.yaml

@@ -0,0 +1,97 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: karpenter
+  namespace: HCP_NAMESPACE
+  labels:
+    app: karpenter
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: karpenter
+  template:
+    metadata:
+      labels:
+        app: karpenter
+    spec:
+      serviceAccountName: karpenter
+      terminationGracePeriodSeconds: 10
+      tolerations:
+        - key: "node-role.kubernetes.io/master"
+          effect: "NoSchedule"
+      volumes:
+        - name: target-kubeconfig
+          secret:
+            # secretName: to be replaced by adaptDeployment
+            defaultMode: 0640
+            items:
+              - key: value
+                path: target-kubeconfig
+        - name: serviceaccount-token
+          emptyDir: {}
+        - name: provider-creds
+          secret:
+            secretName: "karpenter-credentials"
+      containers:
+        - name: karpenter
+          image: karpenter # replaced by payload
+          args:
+            - "--log-level=debug"
+          env:
+            - name: KUBECONFIG
+              value: "/mnt/kubeconfig/target-kubeconfig"
+            - name: SYSTEM_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: DISABLE_WEBHOOK
+              value: "true"
+            - name: DISABLE_LEADER_ELECTION
+              value: "true"
+            - name: FEATURE_GATES
+              value: "Drift=true"
+            - name: AWS_SHARED_CREDENTIALS_FILE
+              value: "/etc/provider/credentials"
+            - name: AWS_SDK_LOAD_CONFIG
+              value: "true"
+            - name: HEALTH_PROBE_PORT
+              value: "8081"
+            - name: CLUSTER_ENDPOINT
+              value: "https://fake.com"
+          ports:
+            - name: metrics
+              containerPort: 8080
+            - name: http
+              containerPort: 8081
+              protocol: TCP
+          resources:
+            requests:
+              memory: "60Mi"
+              cpu: "10m"
+          volumeMounts:
+            - name: target-kubeconfig
+              mountPath: "/mnt/kubeconfig"
+            - name: provider-creds
+              mountPath: "/etc/provider"
+            - name: serviceaccount-token
+              mountPath: "/var/run/secrets/openshift/serviceaccount"
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: http
+              scheme: HTTP
+            initialDelaySeconds: 60
+            periodSeconds: 60
+            successThreshold: 1
+            failureThreshold: 5
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: http
+              scheme: HTTP
+            periodSeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+            timeoutSeconds: 5

control-plane-operator/controllers/hostedcontrolplane/v2/assets/karpenter/role.yaml

@@ -0,0 +1,23 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: karpenter
+  namespace: HCP_NAMESPACE
+rules:
+  - apiGroups:
+      - "coordination.k8s.io"
+    resources:
+      - "leases"
+    verbs:
+      - "get"
+      - "watch"
+      - "create"
+  - apiGroups:
+      - "coordination.k8s.io"
+    resources:
+      - "leases"
+    verbs:
+      - "patch"
+      - "update"
+    resourceNames:
+      - "karpenter-leader-election"

control-plane-operator/controllers/hostedcontrolplane/v2/assets/karpenter/rolebinding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: karpenter
+  namespace: HCP_NAMESPACE
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: karpenter
+subjects:
+- kind: ServiceAccount
+  name: karpenter

control-plane-operator/controllers/hostedcontrolplane/v2/assets/karpenter/serviceacccount.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: karpenter
+  namespace: HCP_NAMESPACE

control-plane-operator/controllers/hostedcontrolplane/v2/karpenter/component.go

@@ -0,0 +1,65 @@
+package karpenter
+
+import (
+	"fmt"
+
+	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests"
+	karpenteroperatorv2 "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/karpenteroperator"
+	component "github.com/openshift/hypershift/support/controlplane-component"
+	"github.com/openshift/hypershift/support/util"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	ComponentName = "karpenter"
+)
+
+var _ component.ComponentOptions = &karpenterOptions{}
+
+type karpenterOptions struct{}
+
+// IsRequestServing implements controlplanecomponent.ComponentOptions.
+func (c *karpenterOptions) IsRequestServing() bool {
+	return false
+}
+
+// MultiZoneSpread implements controlplanecomponent.ComponentOptions.
+func (c *karpenterOptions) MultiZoneSpread() bool {
+	return false
+}
+
+// NeedsManagementKASAccess implements controlplanecomponent.ComponentOptions.
+func (c *karpenterOptions) NeedsManagementKASAccess() bool {
+	return true
+}
+
+func NewComponent() component.ControlPlaneComponent {
+	return component.NewDeploymentComponent(ComponentName, &karpenterOptions{}).
+		WithAdaptFunction(adaptDeployment).
+		WithManifestAdapter(
+			"role.yaml",
+			component.WithAdaptFunction(adaptRole),
+		).
+		WithPredicate(predicate).
+		WithDependencies(karpenteroperatorv2.ComponentName).
+		InjectAvailabilityProberContainer(util.AvailabilityProberOpts{}).
+		Build()
+}
+
+func predicate(cpContext component.WorkloadContext) (bool, error) {
+	hcp := cpContext.HCP
+
+	// The deployment depends on the kubeconfig being reported.
+	if hcp.Status.KubeConfig == nil {
+		return false, nil
+	}
+	// Resolve the kubeconfig secret for CAPI which the autoscaler is deployed alongside of.
+	capiKubeConfigSecret := manifests.KASServiceCAPIKubeconfigSecret(hcp.Namespace, hcp.Spec.InfraID)
+	err := cpContext.Client.Get(cpContext, client.ObjectKeyFromObject(capiKubeConfigSecret), capiKubeConfigSecret)
+	if err != nil {
+		return false, fmt.Errorf("failed to get hosted controlplane kubeconfig secret %q: %w", capiKubeConfigSecret.Name, err)
+	}
+
+	return true, nil
+}

control-plane-operator/controllers/hostedcontrolplane/v2/karpenter/deployment.go

@@ -0,0 +1,92 @@
+package karpenter
+
+import (
+	hyperkarpenterv1 "github.com/openshift/hypershift/api/karpenter/v1beta1"
+	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests"
+	"github.com/openshift/hypershift/support/util"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/utils/ptr"
+
+	component "github.com/openshift/hypershift/support/controlplane-component"
+
+	appsv1 "k8s.io/api/apps/v1"
+)
+
+const (
+	kubeconfigVolumeName = "target-kubeconfig"
+)
+
+func adaptDeployment(cpContext component.WorkloadContext, deployment *appsv1.Deployment) error {
+	hcp := cpContext.HCP
+
+	karpenterProviderAWSOverride, exists := hcp.Annotations[hyperkarpenterv1.KarpenterProviderAWSImage]
+	if exists {
+		util.UpdateContainer(ComponentName, deployment.Spec.Template.Spec.Containers, func(c *corev1.Container) {
+			c.Image = karpenterProviderAWSOverride
+		})
+	}
+
+	util.UpdateVolume(kubeconfigVolumeName, deployment.Spec.Template.Spec.Volumes, func(v *corev1.Volume) {
+		v.Secret.SecretName = manifests.KASServiceCAPIKubeconfigSecret(hcp.Namespace, hcp.Spec.InfraID).Name
+	})
+
+	util.UpdateContainer(ComponentName, deployment.Spec.Template.Spec.Containers, func(c *corev1.Container) {
+		c.Env = append(c.Env,
+			corev1.EnvVar{
+				Name:  "AWS_REGION",
+				Value: hcp.Spec.Platform.AWS.Region,
+			},
+			corev1.EnvVar{
+				Name:  "CLUSTER_NAME",
+				Value: hcp.Spec.InfraID,
+			},
+		)
+		// Override the image if specified in the HCP annotations.
+		karpenterProviderAWSOverride, exists := hcp.Annotations[hyperkarpenterv1.KarpenterProviderAWSImage]
+		if exists {
+			c.Image = karpenterProviderAWSOverride
+		}
+	})
+
+	deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers,
+		corev1.Container{
+			Name:    "token-minter",
+			Image:   "token-minter",
+			Command: []string{"/usr/bin/control-plane-operator", "token-minter"},
+			Args: []string{
+				"--service-account-namespace=kube-system",
+				"--service-account-name=karpenter",
+				"--token-file=/var/run/secrets/openshift/serviceaccount/token",
+				"--kubeconfig=/mnt/kubeconfig/target-kubeconfig",
+			},
+			Resources: corev1.ResourceRequirements{
+				Requests: corev1.ResourceList{
+					corev1.ResourceCPU:    resource.MustParse("10m"),
+					corev1.ResourceMemory: resource.MustParse("30Mi"),
+				},
+			},
+			RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways),
+			StartupProbe: &corev1.Probe{
+				ProbeHandler: corev1.ProbeHandler{
+					Exec: &corev1.ExecAction{
+						Command: []string{"cat", "/var/run/secrets/openshift/serviceaccount/token"}, // waits until the token file is created.
+					},
+				},
+				FailureThreshold: 10,
+			},
+			VolumeMounts: []corev1.VolumeMount{
+				{
+					Name:      "target-kubeconfig",
+					MountPath: "/mnt/kubeconfig",
+				},
+				{
+					Name:      "serviceaccount-token",
+					MountPath: "/var/run/secrets/openshift/serviceaccount",
+				},
+			},
+		},
+	)
+
+	return nil
+}

control-plane-operator/controllers/hostedcontrolplane/v2/karpenter/role.go

@@ -0,0 +1,15 @@
+package karpenter
+
+import (
+	"github.com/openshift/hypershift/support/config"
+	component "github.com/openshift/hypershift/support/controlplane-component"
+
+	rbacv1 "k8s.io/api/rbac/v1"
+)
+
+func adaptRole(cpContext component.WorkloadContext, role *rbacv1.Role) error {
+	ownerRef := config.OwnerRefFrom(cpContext.HCP)
+	ownerRef.ApplyTo(role)
+
+	return nil
+}

control-plane-operator/controllers/hostedcontrolplane/v2/karpenteroperator/component.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests"
 	component "github.com/openshift/hypershift/support/controlplane-component"
+	"github.com/openshift/hypershift/support/util"
 
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -18,8 +19,6 @@ var _ component.ComponentOptions = &KarpenterOperatorOptions{}
 type KarpenterOperatorOptions struct {
 	HyperShiftOperatorImage   string
 	ControlPlaneOperatorImage string
-	KarpenterProviderAWSImage string
-	ControlPlaneContext       component.ControlPlaneContext
 }
 
 // IsRequestServing implements controlplanecomponent.ComponentOptions.
@@ -51,6 +50,7 @@ func NewComponent(options *KarpenterOperatorOptions) component.ControlPlaneCompo
 			ServiceAccountNameSpace: "kube-system",
 			KubeconfigSecretName:    "service-network-admin-kubeconfig",
 		}).
+		InjectAvailabilityProberContainer(util.AvailabilityProberOpts{}).
 		Build()
 }
 

control-plane-operator/controllers/hostedcontrolplane/v2/karpenteroperator/deployment.go

@@ -53,7 +53,6 @@ func (karp *KarpenterOperatorOptions) adaptDeployment(cpContext component.Worklo
 			)
 			c.Args = append(c.Args,
 				"--control-plane-operator-image="+karp.ControlPlaneOperatorImage,
-				"--karpenter-provider-aws-image="+karp.KarpenterProviderAWSImage,
 			)
 		})
 	}

hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go

@@ -1881,6 +1881,7 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques
 
 	imageProvider := imageprovider.New(releaseImage)
 	imageProvider.ComponentImages()["token-minter"] = utilitiesImage
+	imageProvider.ComponentImages()[hyperutil.AvailabilityProberImageName] = utilitiesImage
 	cpContext := controlplanecomponent.ControlPlaneContext{
 		Context:                   ctx,
 		Client:                    r.Client,

hypershift-operator/controllers/hostedcluster/karpenter.go

@@ -20,7 +20,6 @@ import (
 	karpenteroperatorv2 "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/karpenteroperator"
 	"github.com/openshift/hypershift/hypershift-operator/controllers/nodepool"
 	haproxy "github.com/openshift/hypershift/hypershift-operator/controllers/nodepool/apiserver-haproxy"
-	"github.com/openshift/hypershift/karpenter-operator/controllers/karpenter/assets"
 	controlplanecomponent "github.com/openshift/hypershift/support/controlplane-component"
 
 	"github.com/openshift/hypershift/support/releaseinfo"
@@ -109,16 +108,9 @@ spec:
 
 	// Run karpenter Operator to manage CRs management and guest side.
 
-	// TODO(jkyros): Grab the karpenter image in the proper place at the beginning with args, not here?
-	karpenterProviderAWSImage, hasImage := releaseImage.ComponentImages()["aws-karpenter-provider-aws"]
-	if !hasImage {
-		karpenterProviderAWSImage = assets.DefaultKarpenterProviderAWSImage
-	}
-
 	karpenteroperator := karpenteroperatorv2.NewComponent(&karpenteroperatorv2.KarpenterOperatorOptions{
 		HyperShiftOperatorImage:   hypershiftOperatorImage,
 		ControlPlaneOperatorImage: controlPlaneOperatorImage,
-		KarpenterProviderAWSImage: karpenterProviderAWSImage,
 	})
 
 	if err := karpenteroperator.Reconcile(cpContext); err != nil {