controller/registry: implement content extraction for catalog sources

Signed-off-by: Steve Kuznetsov <[email protected]>

Author: stevekuznetsov

deploy/chart/crds/0000_50_olm_00-catalogsources.crd.yaml

@@ -532,8 +532,18 @@ spec:
                                   topologyKey:
                                     description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.
                                     type: string
+                    extractContent:
+                      description: ExtractContent configures the gRPC catalog Pod to extract catalog metadata from the provided index image and use a well-known version of the `opm` server to expose it. The catalog index image that this CatalogSource is configured to use *must* be using the file-based catalogs in order to utilize this feature.
+                      type: object
+                      properties:
+                        cacheDir:
+                          description: CacheDir is the directory storing the pre-calculated API cache.
+                          type: string
+                        catalogDir:
+                          description: CatalogDir is the directory storing the file-based catalog contents.
+                          type: string
                     memoryTarget:
-                      description: "MemoryTarget configures the $GOMEMLIMIT value for the gRPC catalog Pod. This is a soft memory limit for the server, which the runtime will attempt to meet but makes no guarantees that it will do so. If this value is set, the Pod will have the following modifications made to the container running the server: - the $GOMEMLIMIT environment variable will be set to this value in bytes - the memory request will be set to this value - the memory limit will be set to 200% of this value \n This field should be set if it's desired to reduce the footprint of a catalog server as much as possible, or if a catalog being served is very large and needs more than the default allocation. If your index image has a file- system cache, determine a good approximation for this value by doubling the size of the package cache at /tmp/cache/cache/packages.json in the index image. \n This field is best-effort; if unset, no default will be used and no Pod memory limit or $GOMEMLIMIT value will be set."
+                      description: "MemoryTarget configures the $GOMEMLIMIT value for the gRPC catalog Pod. This is a soft memory limit for the server, which the runtime will attempt to meet but makes no guarantees that it will do so. If this value is set, the Pod will have the following modifications made to the container running the server: - the $GOMEMLIMIT environment variable will be set to this value in bytes - the memory request will be set to this value \n This field should be set if it's desired to reduce the footprint of a catalog server as much as possible, or if a catalog being served is very large and needs more than the default allocation. If your index image has a file- system cache, determine a good approximation for this value by doubling the size of the package cache at /tmp/cache/cache/packages.json in the index image. \n This field is best-effort; if unset, no default will be used and no Pod memory limit or $GOMEMLIMIT value will be set."
                       pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                       anyOf:
                         - type: integer

pkg/controller/operators/catalog/operator.go

@@ -217,7 +217,7 @@ func NewOperator(ctx context.Context, kubeconfigPath string, clock utilclock.Clo
 	op.sources = grpc.NewSourceStore(logger, 10*time.Second, 10*time.Minute, op.syncSourceState)
 	op.sourceInvalidator = resolver.SourceProviderFromRegistryClientProvider(op.sources, logger)
 	resolverSourceProvider := NewOperatorGroupToggleSourceProvider(op.sourceInvalidator, logger, op.lister.OperatorsV1().OperatorGroupLister())
-	op.reconciler = reconciler.NewRegistryReconcilerFactory(lister, opClient, configmapRegistryImage, op.now, ssaClient, workloadUserID)
+	op.reconciler = reconciler.NewRegistryReconcilerFactory(lister, opClient, configmapRegistryImage, op.now, ssaClient, workloadUserID, opmImage)
 	res := resolver.NewOperatorStepResolver(lister, crClient, operatorNamespace, resolverSourceProvider, logger)
 	op.resolver = resolver.NewInstrumentedResolver(res, metrics.RegisterDependencyResolutionSuccess, metrics.RegisterDependencyResolutionFailure)
 

pkg/controller/operators/catalog/operator_test.go

@@ -1773,7 +1773,7 @@ func NewFakeOperator(ctx context.Context, namespace string, namespaces []string,
 		}
 		applier := controllerclient.NewFakeApplier(s, "testowner")
 
-		op.reconciler = reconciler.NewRegistryReconcilerFactory(lister, op.opClient, "test:pod", op.now, applier, 1001)
+		op.reconciler = reconciler.NewRegistryReconcilerFactory(lister, op.opClient, "test:pod", op.now, applier, 1001, "")
 	}
 
 	op.RunInformers(ctx)
@@ -1929,7 +1929,7 @@ func toManifest(t *testing.T, obj runtime.Object) string {
 }
 
 func pod(s v1alpha1.CatalogSource) *corev1.Pod {
-	pod := reconciler.Pod(&s, "registry-server", s.Spec.Image, s.GetName(), s.GetLabels(), s.GetAnnotations(), 5, 10, 1001)
+	pod := reconciler.Pod(&s, "registry-server", "central-opm", s.Spec.Image, s.GetName(), s.GetLabels(), s.GetAnnotations(), 5, 10, 1001)
 	ownerutil.AddOwner(pod, &s, false, true)
 	return pod
 }

pkg/controller/registry/reconciler/configmap.go

@@ -106,7 +106,7 @@ func (s *configMapCatalogSourceDecorator) Service() *corev1.Service {
 }
 
 func (s *configMapCatalogSourceDecorator) Pod(image string) *corev1.Pod {
-	pod := Pod(s.CatalogSource, "configmap-registry-server", image, "", s.Labels(), s.Annotations(), 5, 5, s.runAsUser)
+	pod := Pod(s.CatalogSource, "configmap-registry-server", "", image, "", s.Labels(), s.Annotations(), 5, 5, s.runAsUser)
 	pod.Spec.ServiceAccountName = s.GetName() + ConfigMapServerPostfix
 	pod.Spec.Containers[0].Command = []string{"configmap-server", "-c", s.Spec.ConfigMap, "-n", s.GetNamespace()}
 	ownerutil.AddOwner(pod, s.CatalogSource, false, true)

pkg/controller/registry/reconciler/configmap_test.go

@@ -198,7 +198,7 @@ func objectsForCatalogSource(catsrc *v1alpha1.CatalogSource) []runtime.Object {
 		)
 	case v1alpha1.SourceTypeGrpc:
 		if catsrc.Spec.Image != "" {
-			decorated := grpcCatalogSourceDecorator{catsrc, runAsUser}
+			decorated := grpcCatalogSourceDecorator{CatalogSource: catsrc, createPodAsUser: runAsUser, opmImage: ""}
 			objs = clientfake.AddSimpleGeneratedNames(
 				decorated.Pod(catsrc.GetName()),
 				decorated.Service(),

pkg/controller/registry/reconciler/grpc.go

@@ -35,6 +35,7 @@ const (
 type grpcCatalogSourceDecorator struct {
 	*v1alpha1.CatalogSource
 	createPodAsUser int64
+	opmImage        string
 }
 
 type UpdateNotReadyErr struct {
@@ -128,7 +129,7 @@ func (s *grpcCatalogSourceDecorator) ServiceAccount() *corev1.ServiceAccount {
 }
 
 func (s *grpcCatalogSourceDecorator) Pod(saName string) *corev1.Pod {
-	pod := Pod(s.CatalogSource, "registry-server", s.Spec.Image, saName, s.Labels(), s.Annotations(), 5, 10, s.createPodAsUser)
+	pod := Pod(s.CatalogSource, "registry-server", s.opmImage, s.Spec.Image, saName, s.Labels(), s.Annotations(), 5, 10, s.createPodAsUser)
 	ownerutil.AddOwner(pod, s.CatalogSource, false, true)
 	return pod
 }
@@ -139,6 +140,7 @@ type GrpcRegistryReconciler struct {
 	OpClient        operatorclient.ClientInterface
 	SSAClient       *controllerclient.ServerSideApplier
 	createPodAsUser int64
+	opmImage        string
 }
 
 var _ RegistryReconciler = &GrpcRegistryReconciler{}
@@ -196,16 +198,25 @@ func (c *GrpcRegistryReconciler) currentPodsWithCorrectImageAndSpec(source grpcC
 	found := []*corev1.Pod{}
 	newPod := source.Pod(saName)
 	for _, p := range pods {
-		if p.Spec.Containers[0].Image == source.Spec.Image && podHashMatch(p, newPod) {
+		if correctImages(source, p) && podHashMatch(p, newPod) {
 			found = append(found, p)
 		}
 	}
 	return found
 }
 
+func correctImages(source grpcCatalogSourceDecorator, pod *corev1.Pod) bool {
+	if source.CatalogSource.Spec.GrpcPodConfig != nil && source.CatalogSource.Spec.GrpcPodConfig.ExtractContent != nil {
+		return pod.Spec.InitContainers[0].Image == source.opmImage &&
+			pod.Spec.InitContainers[1].Image == source.CatalogSource.Spec.Image &&
+			pod.Spec.Containers[0].Image == source.opmImage
+	}
+	return pod.Spec.Containers[0].Image == source.CatalogSource.Spec.Image
+}
+
 // EnsureRegistryServer ensures that all components of registry server are up to date.
 func (c *GrpcRegistryReconciler) EnsureRegistryServer(catalogSource *v1alpha1.CatalogSource) error {
-	source := grpcCatalogSourceDecorator{catalogSource, c.createPodAsUser}
+	source := grpcCatalogSourceDecorator{CatalogSource: catalogSource, createPodAsUser: c.createPodAsUser, opmImage: c.opmImage}
 
 	// if service status is nil, we force create every object to ensure they're created the first time
 	overwrite := source.Status.RegistryServiceStatus == nil || !isRegistryServiceStatusValid(&source)
@@ -454,7 +465,7 @@ func (c *GrpcRegistryReconciler) removePods(pods []*corev1.Pod, namespace string
 
 // CheckRegistryServer returns true if the given CatalogSource is considered healthy; false otherwise.
 func (c *GrpcRegistryReconciler) CheckRegistryServer(catalogSource *v1alpha1.CatalogSource) (healthy bool, err error) {
-	source := grpcCatalogSourceDecorator{catalogSource, c.createPodAsUser}
+	source := grpcCatalogSourceDecorator{CatalogSource: catalogSource, createPodAsUser: c.createPodAsUser, opmImage: c.opmImage}
 	// Check on registry resources
 	// TODO: add gRPC health check
 	if len(c.currentPodsWithCorrectImageAndSpec(source, source.ServiceAccount().GetName())) < 1 ||

pkg/controller/registry/reconciler/grpc_test.go

@@ -2,9 +2,11 @@ package reconciler
 
 import (
 	"context"
+	"fmt"
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/install"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
@@ -355,7 +357,7 @@ func TestGrpcRegistryReconciler(t *testing.T) {
 			}
 
 			// Check for resource existence
-			decorated := grpcCatalogSourceDecorator{tt.in.catsrc, runAsUser}
+			decorated := grpcCatalogSourceDecorator{CatalogSource: tt.in.catsrc, createPodAsUser: runAsUser}
 			pod := decorated.Pod(tt.in.catsrc.GetName())
 			service := decorated.Service()
 			sa := decorated.ServiceAccount()
@@ -367,6 +369,9 @@ func TestGrpcRegistryReconciler(t *testing.T) {
 			case *GrpcRegistryReconciler:
 				// Should be created by a GrpcRegistryReconciler
 				require.NoError(t, podErr)
+				if diff := cmp.Diff(outPods.Items, []corev1.Pod{*pod}); diff != "" {
+					fmt.Printf("incorrect pods: %s\n", diff)
+				}
 				require.Len(t, outPods.Items, 1)
 				outPod := outPods.Items[0]
 				require.Equal(t, pod.GetGenerateName(), outPod.GetGenerateName())
@@ -445,7 +450,7 @@ func TestRegistryPodPriorityClass(t *testing.T) {
 			require.NoError(t, err)
 
 			// Check for resource existence
-			decorated := grpcCatalogSourceDecorator{tt.in.catsrc, runAsUser}
+			decorated := grpcCatalogSourceDecorator{CatalogSource: tt.in.catsrc, createPodAsUser: runAsUser}
 			pod := decorated.Pod(tt.in.catsrc.GetName())
 			listOptions := metav1.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set{CatalogSourceLabelKey: tt.in.catsrc.GetName()}).String()}
 			outPods, podErr := client.KubernetesInterface().CoreV1().Pods(pod.GetNamespace()).List(context.TODO(), listOptions)

pkg/controller/registry/reconciler/reconciler.go

@@ -4,6 +4,7 @@ package reconciler
 import (
 	"fmt"
 	"hash/fnv"
+	"path/filepath"
 
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/install"
 	corev1 "k8s.io/api/core/v1"
@@ -64,6 +65,7 @@ type registryReconcilerFactory struct {
 	ConfigMapServerImage string
 	SSAClient            *controllerclient.ServerSideApplier
 	createPodAsUser      int64
+	opmImage             string
 }
 
 // ReconcilerForSource returns a RegistryReconciler based on the configuration of the given CatalogSource.
@@ -86,6 +88,7 @@ func (r *registryReconcilerFactory) ReconcilerForSource(source *operatorsv1alpha
 				OpClient:        r.OpClient,
 				SSAClient:       r.SSAClient,
 				createPodAsUser: r.createPodAsUser,
+				opmImage:        r.opmImage,
 			}
 		} else if source.Spec.Address != "" {
 			return &GrpcAddressRegistryReconciler{
@@ -97,18 +100,19 @@ func (r *registryReconcilerFactory) ReconcilerForSource(source *operatorsv1alpha
 }
 
 // NewRegistryReconcilerFactory returns an initialized RegistryReconcilerFactory.
-func NewRegistryReconcilerFactory(lister operatorlister.OperatorLister, opClient operatorclient.ClientInterface, configMapServerImage string, now nowFunc, ssaClient *controllerclient.ServerSideApplier, createPodAsUser int64) RegistryReconcilerFactory {
+func NewRegistryReconcilerFactory(lister operatorlister.OperatorLister, opClient operatorclient.ClientInterface, configMapServerImage string, now nowFunc, ssaClient *controllerclient.ServerSideApplier, createPodAsUser int64, opmImage string) RegistryReconcilerFactory {
 	return &registryReconcilerFactory{
 		now:                  now,
 		Lister:               lister,
 		OpClient:             opClient,
 		ConfigMapServerImage: configMapServerImage,
 		SSAClient:            ssaClient,
 		createPodAsUser:      createPodAsUser,
+		opmImage:             opmImage,
 	}
 }
 
-func Pod(source *operatorsv1alpha1.CatalogSource, name string, img string, saName string, labels map[string]string, annotations map[string]string, readinessDelay int32, livenessDelay int32, runAsUser int64) *corev1.Pod {
+func Pod(source *operatorsv1alpha1.CatalogSource, name, opmImg, img, saName string, labels, annotations map[string]string, readinessDelay, livenessDelay int32, runAsUser int64) *corev1.Pod {
 	// make a copy of the labels and annotations to avoid mutating the input parameters
 	podLabels := make(map[string]string)
 	podAnnotations := make(map[string]string)
@@ -236,6 +240,58 @@ func Pod(source *operatorsv1alpha1.CatalogSource, name string, img string, saNam
 				Value: grpcPodConfig.MemoryTarget.String() + "B", // k8s resources use Mi, GOMEMLIMIT wants MiB
 			})
 		}
+
+		// Reconfigure pod to extract content
+		if grpcPodConfig.ExtractContent != nil {
+			pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{
+				Name: "utilities",
+				VolumeSource: corev1.VolumeSource{
+					EmptyDir: &corev1.EmptyDirVolumeSource{},
+				},
+			}, corev1.Volume{
+				Name: "catalog-content",
+				VolumeSource: corev1.VolumeSource{
+					EmptyDir: &corev1.EmptyDirVolumeSource{},
+				},
+			})
+			const utilitiesPath = "/utilities"
+			utilitiesVolumeMount := corev1.VolumeMount{
+				Name:      "utilities",
+				MountPath: utilitiesPath,
+			}
+			const catalogPath = "/extracted-catalog"
+			contentVolumeMount := corev1.VolumeMount{
+				Name:      "catalog-content",
+				MountPath: catalogPath,
+			}
+			pod.Spec.InitContainers = append(pod.Spec.InitContainers, corev1.Container{
+				Name:    "extract-utilities",
+				Image:   opmImg,
+				Command: []string{"sh", "-c"},
+				Args: []string{fmt.Sprintf("cp $( command -v sh ) %s/sh && cp $( command -v cp ) %s/cp",
+					utilitiesPath, utilitiesPath,
+				)},
+				VolumeMounts: []corev1.VolumeMount{utilitiesVolumeMount},
+			}, corev1.Container{
+				Name:    "extract-content",
+				Image:   img,
+				Command: []string{utilitiesPath + "/sh", "-c"},
+				Args: []string{fmt.Sprintf("%s/cp -r %s %s/catalog && %s/cp -r %s %s/cache",
+					utilitiesPath, grpcPodConfig.ExtractContent.CatalogDir, catalogPath,
+					utilitiesPath, grpcPodConfig.ExtractContent.CacheDir, catalogPath,
+				)},
+				VolumeMounts: []corev1.VolumeMount{utilitiesVolumeMount, contentVolumeMount},
+			})
+
+			pod.Spec.Containers[0].Image = opmImg
+			pod.Spec.Containers[0].Command = []string{"/bin/opm"}
+			pod.Spec.Containers[0].Args = []string{
+				"serve",
+				filepath.Join(catalogPath, "catalog"),
+				"--cache-dir=" + filepath.Join(catalogPath, "cache"),
+			}
+			pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, contentVolumeMount)
+		}
 	}
 
 	// Set priorityclass if its annotation exists