Merge pull request #15672 from deads2k/tsb-06-separate

Automatic merge from submit-queue

Run the TSB in a separate pod

This completes the work necessary to run the TSB in a pod.  We should continue to remove the cruft from the old path, but I've included a template which creates a running TSB server in the `openshift-template-service-broker` namespace.

@bparees @jim-minter

Author: openshift-merge-robot

contrib/completions/bash/openshift

@@ -38516,30 +38516,8 @@ _openshift_start_template-service-broker()
     local_nonpersistent_flags+=("--client-ca-file=")
     flags+=("--contention-profiling")
     local_nonpersistent_flags+=("--contention-profiling")
-    flags+=("--delete-collection-workers=")
-    local_nonpersistent_flags+=("--delete-collection-workers=")
-    flags+=("--deserialization-cache-size=")
-    local_nonpersistent_flags+=("--deserialization-cache-size=")
-    flags+=("--enable-garbage-collector")
-    local_nonpersistent_flags+=("--enable-garbage-collector")
     flags+=("--enable-swagger-ui")
     local_nonpersistent_flags+=("--enable-swagger-ui")
-    flags+=("--etcd-cafile=")
-    local_nonpersistent_flags+=("--etcd-cafile=")
-    flags+=("--etcd-certfile=")
-    local_nonpersistent_flags+=("--etcd-certfile=")
-    flags+=("--etcd-keyfile=")
-    local_nonpersistent_flags+=("--etcd-keyfile=")
-    flags+=("--etcd-prefix=")
-    local_nonpersistent_flags+=("--etcd-prefix=")
-    flags+=("--etcd-quorum-read")
-    local_nonpersistent_flags+=("--etcd-quorum-read")
-    flags+=("--etcd-servers=")
-    local_nonpersistent_flags+=("--etcd-servers=")
-    flags+=("--etcd-servers-overrides=")
-    local_nonpersistent_flags+=("--etcd-servers-overrides=")
-    flags+=("--experimental-encryption-provider-config=")
-    local_nonpersistent_flags+=("--experimental-encryption-provider-config=")
     flags+=("--profiling")
     local_nonpersistent_flags+=("--profiling")
     flags+=("--requestheader-allowed-names=")
@@ -38554,10 +38532,6 @@ _openshift_start_template-service-broker()
     local_nonpersistent_flags+=("--requestheader-username-headers=")
     flags+=("--secure-port=")
     local_nonpersistent_flags+=("--secure-port=")
-    flags+=("--storage-backend=")
-    local_nonpersistent_flags+=("--storage-backend=")
-    flags+=("--storage-media-type=")
-    local_nonpersistent_flags+=("--storage-media-type=")
     flags+=("--template-namespace=")
     local_nonpersistent_flags+=("--template-namespace=")
     flags+=("--tls-ca-file=")
@@ -38568,8 +38542,6 @@ _openshift_start_template-service-broker()
     local_nonpersistent_flags+=("--tls-private-key-file=")
     flags+=("--tls-sni-cert-key=")
     local_nonpersistent_flags+=("--tls-sni-cert-key=")
-    flags+=("--watch-cache")
-    local_nonpersistent_flags+=("--watch-cache")
     flags+=("--azure-container-registry-config=")
     flags+=("--google-json-key=")
     flags+=("--log-flush-frequency=")

contrib/completions/zsh/openshift

@@ -38665,30 +38665,8 @@ _openshift_start_template-service-broker()
     local_nonpersistent_flags+=("--client-ca-file=")
     flags+=("--contention-profiling")
     local_nonpersistent_flags+=("--contention-profiling")
-    flags+=("--delete-collection-workers=")
-    local_nonpersistent_flags+=("--delete-collection-workers=")
-    flags+=("--deserialization-cache-size=")
-    local_nonpersistent_flags+=("--deserialization-cache-size=")
-    flags+=("--enable-garbage-collector")
-    local_nonpersistent_flags+=("--enable-garbage-collector")
     flags+=("--enable-swagger-ui")
     local_nonpersistent_flags+=("--enable-swagger-ui")
-    flags+=("--etcd-cafile=")
-    local_nonpersistent_flags+=("--etcd-cafile=")
-    flags+=("--etcd-certfile=")
-    local_nonpersistent_flags+=("--etcd-certfile=")
-    flags+=("--etcd-keyfile=")
-    local_nonpersistent_flags+=("--etcd-keyfile=")
-    flags+=("--etcd-prefix=")
-    local_nonpersistent_flags+=("--etcd-prefix=")
-    flags+=("--etcd-quorum-read")
-    local_nonpersistent_flags+=("--etcd-quorum-read")
-    flags+=("--etcd-servers=")
-    local_nonpersistent_flags+=("--etcd-servers=")
-    flags+=("--etcd-servers-overrides=")
-    local_nonpersistent_flags+=("--etcd-servers-overrides=")
-    flags+=("--experimental-encryption-provider-config=")
-    local_nonpersistent_flags+=("--experimental-encryption-provider-config=")
     flags+=("--profiling")
     local_nonpersistent_flags+=("--profiling")
     flags+=("--requestheader-allowed-names=")
@@ -38703,10 +38681,6 @@ _openshift_start_template-service-broker()
     local_nonpersistent_flags+=("--requestheader-username-headers=")
     flags+=("--secure-port=")
     local_nonpersistent_flags+=("--secure-port=")
-    flags+=("--storage-backend=")
-    local_nonpersistent_flags+=("--storage-backend=")
-    flags+=("--storage-media-type=")
-    local_nonpersistent_flags+=("--storage-media-type=")
     flags+=("--template-namespace=")
     local_nonpersistent_flags+=("--template-namespace=")
     flags+=("--tls-ca-file=")
@@ -38717,8 +38691,6 @@ _openshift_start_template-service-broker()
     local_nonpersistent_flags+=("--tls-private-key-file=")
     flags+=("--tls-sni-cert-key=")
     local_nonpersistent_flags+=("--tls-sni-cert-key=")
-    flags+=("--watch-cache")
-    local_nonpersistent_flags+=("--watch-cache")
     flags+=("--azure-container-registry-config=")
     flags+=("--google-json-key=")
     flags+=("--log-flush-frequency=")

pkg/cmd/server/origin/master.go

@@ -97,7 +97,7 @@ func (c *MasterConfig) newOpenshiftNonAPIConfig(kubeAPIServerConfig apiserver.Co
 func (c *MasterConfig) newTemplateServiceBrokerConfig(kubeAPIServerConfig apiserver.Config) *openservicebrokerserver.TemplateServiceBrokerConfig {
 	ret := &openservicebrokerserver.TemplateServiceBrokerConfig{
 		GenericConfig:              &kubeAPIServerConfig,
-		PrivilegedKubeClientConfig: *kubeAPIServerConfig.LoopbackClientConfig,
+		PrivilegedKubeClientConfig: kubeAPIServerConfig.LoopbackClientConfig,
 		TemplateInformers:          c.TemplateInformers,
 		TemplateNamespaces:         c.Options.TemplateServiceBrokerConfig.TemplateNamespaces,
 	}

pkg/openservicebroker/cmd/server/start.go

@@ -13,10 +13,13 @@ import (
 	"github.com/openshift/origin/pkg/openservicebroker/server"
 )
 
-const defaultEtcdPathPrefix = "/registry/templateservicebroker.openshift.io"
-
 type TemplateServiceBrokerServerOptions struct {
-	RecommendedOptions *genericoptions.RecommendedOptions
+	// we don't have any storage, so we shouldn't use the recommended options
+	SecureServing  *genericoptions.SecureServingOptions
+	Authentication *genericoptions.DelegatingAuthenticationOptions
+	Authorization  *genericoptions.DelegatingAuthorizationOptions
+	Audit          *genericoptions.AuditOptions
+	Features       *genericoptions.FeatureOptions
 
 	StdOut io.Writer
 	StdErr io.Writer
@@ -26,7 +29,11 @@ type TemplateServiceBrokerServerOptions struct {
 
 func NewTemplateServiceBrokerServerOptions(out, errOut io.Writer) *TemplateServiceBrokerServerOptions {
 	o := &TemplateServiceBrokerServerOptions{
-		RecommendedOptions: genericoptions.NewRecommendedOptions(defaultEtcdPathPrefix, server.Scheme, server.Codecs.LegacyCodec()),
+		SecureServing:  genericoptions.NewSecureServingOptions(),
+		Authentication: genericoptions.NewDelegatingAuthenticationOptions(),
+		Authorization:  genericoptions.NewDelegatingAuthorizationOptions(),
+		Audit:          genericoptions.NewAuditOptions(),
+		Features:       genericoptions.NewFeatureOptions(),
 
 		StdOut: out,
 		StdErr: errOut,
@@ -57,7 +64,11 @@ func NewCommandStartTemplateServiceBrokerServer(out, errOut io.Writer, stopCh <-
 	}
 
 	flags := cmd.Flags()
-	o.RecommendedOptions.AddFlags(flags)
+	o.SecureServing.AddFlags(flags)
+	o.Authentication.AddFlags(flags)
+	o.Authorization.AddFlags(flags)
+	o.Audit.AddFlags(flags)
+	o.Features.AddFlags(flags)
 	flags.StringSliceVar(&o.TemplateNamespaces, "template-namespace", o.TemplateNamespaces, "TemplateNamespaces indicates the namespace(s) in which the template service broker looks for templates to serve to the catalog.")
 
 	return cmd
@@ -73,12 +84,24 @@ func (o *TemplateServiceBrokerServerOptions) Complete() error {
 
 func (o TemplateServiceBrokerServerOptions) Config() (*server.TemplateServiceBrokerConfig, error) {
 	// TODO have a "real" external address
-	if err := o.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil {
+	if err := o.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil {
 		return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
 	}
 
 	serverConfig := genericapiserver.NewConfig(server.Codecs)
-	if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil {
+	if err := o.SecureServing.ApplyTo(serverConfig); err != nil {
+		return nil, err
+	}
+	if err := o.Authentication.ApplyTo(serverConfig); err != nil {
+		return nil, err
+	}
+	if err := o.Authorization.ApplyTo(serverConfig); err != nil {
+		return nil, err
+	}
+	if err := o.Audit.ApplyTo(serverConfig); err != nil {
+		return nil, err
+	}
+	if err := o.Features.ApplyTo(serverConfig); err != nil {
 		return nil, err
 	}
 

pkg/openservicebroker/server/apiserver.go

@@ -15,12 +15,15 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
 	"k8s.io/kubernetes/pkg/api"
 	kclientsetinternal "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
+	"k8s.io/kubernetes/pkg/controller"
 
 	"github.com/openshift/origin/pkg/cmd/server/bootstrappolicy"
 	templateapi "github.com/openshift/origin/pkg/template/apis/template"
 	templateinformer "github.com/openshift/origin/pkg/template/generated/informers/internalversion"
+	templateinternalclientset "github.com/openshift/origin/pkg/template/generated/internalclientset"
 	templateservicebroker "github.com/openshift/origin/pkg/template/servicebroker"
 )
 
@@ -51,7 +54,7 @@ type TemplateServiceBrokerConfig struct {
 
 	// PrivilegedKubeClientConfig is *not* a loopback config, since it needs to point to the kube apiserver
 	// TODO remove this and use the SA that start us instead of trying to cyclically find an SA token
-	PrivilegedKubeClientConfig restclient.Config
+	PrivilegedKubeClientConfig *restclient.Config
 
 	TemplateInformers  templateinformer.SharedInformerFactory
 	TemplateNamespaces []string
@@ -87,11 +90,55 @@ func (c completedTemplateServiceBrokerConfig) New(delegationTarget genericapiser
 		GenericAPIServer: genericServer,
 	}
 
-	broker := templateservicebroker.DeprecatedNewBrokerInsideAPIServer(
-		c.PrivilegedKubeClientConfig,
-		c.TemplateInformers.Template().InternalVersion().Templates(),
-		c.TemplateNamespaces,
-	)
+	inCluster := false
+	var broker *templateservicebroker.Broker
+	// TODO, this block drops out after the server is moved.
+	if c.PrivilegedKubeClientConfig != nil {
+		broker = templateservicebroker.DeprecatedNewBrokerInsideAPIServer(
+			*c.PrivilegedKubeClientConfig,
+			c.TemplateInformers.Template().InternalVersion().Templates(),
+			c.TemplateNamespaces,
+		)
+		// make sure no one else uses it
+		c.TemplateInformers = nil
+
+	} else {
+		// we're running in cluster
+		// in this case we actually want to construct our own template informer
+		// eventually the config value drops out as it isn't supported server side
+		inCluster = true
+		clientConfig, err := restclient.InClusterConfig()
+		if err != nil {
+			return nil, err
+		}
+		templateClient, err := templateinternalclientset.NewForConfig(clientConfig)
+		if err != nil {
+			return nil, err
+		}
+		templateInformers := templateinformer.NewSharedInformerFactory(templateClient, 5*time.Minute)
+		templateInformers.Template().InternalVersion().Templates().Informer().AddIndexers(cache.Indexers{
+			templateapi.TemplateUIDIndex: func(obj interface{}) ([]string, error) {
+				return []string{string(obj.(*templateapi.Template).UID)}, nil
+			},
+		})
+
+		broker, err = templateservicebroker.NewBroker(
+			clientConfig,
+			templateInformers.Template().InternalVersion().Templates(),
+			c.TemplateNamespaces,
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		s.GenericAPIServer.AddPostStartHook("template-service-broker-synctemplates", func(context genericapiserver.PostStartHookContext) error {
+			templateInformers.Start(context.StopCh)
+			if !controller.WaitForCacheSync("tsb", context.StopCh, templateInformers.Template().InternalVersion().Templates().Informer().HasSynced) {
+				return fmt.Errorf("unable to sync caches")
+			}
+			return nil
+		})
+	}
 
 	Route(
 		s.GenericAPIServer.Handler.GoRestfulContainer,
@@ -100,32 +147,34 @@ func (c completedTemplateServiceBrokerConfig) New(delegationTarget genericapiser
 	)
 
 	// TODO, when the TSB becomes a separate entity, this should stop creating the SA and use its container pod SA identity instead
-	s.GenericAPIServer.AddPostStartHook("template-service-broker-ensure-service-account", func(context genericapiserver.PostStartHookContext) error {
-		kc, err := kclientsetinternal.NewForConfig(context.LoopbackClientConfig)
-		if err != nil {
-			utilruntime.HandleError(fmt.Errorf("template service broker: failed to get client: %v", err))
-			return err
-		}
+	if !inCluster {
+		s.GenericAPIServer.AddPostStartHook("template-service-broker-ensure-service-account", func(context genericapiserver.PostStartHookContext) error {
+			kc, err := kclientsetinternal.NewForConfig(context.LoopbackClientConfig)
+			if err != nil {
+				utilruntime.HandleError(fmt.Errorf("template service broker: failed to get client: %v", err))
+				return err
+			}
 
-		err = wait.PollImmediate(time.Second, 30*time.Second, func() (done bool, err error) {
-			kc.Namespaces().Create(&api.Namespace{ObjectMeta: metav1.ObjectMeta{Name: bootstrappolicy.DefaultOpenShiftInfraNamespace}})
+			err = wait.PollImmediate(time.Second, 30*time.Second, func() (done bool, err error) {
+				kc.Namespaces().Create(&api.Namespace{ObjectMeta: metav1.ObjectMeta{Name: bootstrappolicy.DefaultOpenShiftInfraNamespace}})
 
-			_, err = kc.ServiceAccounts(bootstrappolicy.DefaultOpenShiftInfraNamespace).Create(&api.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: bootstrappolicy.InfraTemplateServiceBrokerServiceAccountName}})
-			switch {
-			case err == nil || kapierrors.IsAlreadyExists(err):
-				done, err = true, nil
-			case kapierrors.IsNotFound(err):
-				err = nil
-			}
+				_, err = kc.ServiceAccounts(bootstrappolicy.DefaultOpenShiftInfraNamespace).Create(&api.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: bootstrappolicy.InfraTemplateServiceBrokerServiceAccountName}})
+				switch {
+				case err == nil || kapierrors.IsAlreadyExists(err):
+					done, err = true, nil
+				case kapierrors.IsNotFound(err):
+					err = nil
+				}
 
-			return
-		})
+				return
+			})
 
-		if err != nil {
-			utilruntime.HandleError(fmt.Errorf("creation of template-service-broker SA failed: %v", err))
-		}
-		return err
-	})
+			if err != nil {
+				utilruntime.HandleError(fmt.Errorf("creation of template-service-broker SA failed: %v", err))
+			}
+			return err
+		})
+	}
 
 	return s, nil
 }

pkg/template/servicebroker/servicebroker.go

@@ -64,6 +64,47 @@ func DeprecatedNewBrokerInsideAPIServer(privilegedKubeClientConfig restclient.Co
 	return b
 }
 
+func NewBroker(saKubeClientConfig *restclient.Config, informer templateinformer.TemplateInformer, namespaces []string) (*Broker, error) {
+	templateNamespaces := map[string]struct{}{}
+	for _, namespace := range namespaces {
+		templateNamespaces[namespace] = struct{}{}
+	}
+
+	internalKubeClient, err := kclientset.NewForConfig(saKubeClientConfig)
+	if err != nil {
+		return nil, err
+	}
+	externalKubeClient, err := kclientsetexternal.NewForConfig(saKubeClientConfig)
+	if err != nil {
+		return nil, err
+	}
+	extrouteclientset, err := extrouteclientset.NewForConfig(saKubeClientConfig)
+	if err != nil {
+		return nil, err
+	}
+	templateClient, err := templateclientset.NewForConfig(saKubeClientConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	b := &Broker{
+		kc:                 internalKubeClient,
+		extkc:              externalKubeClient,
+		extrouteclient:     extrouteclientset,
+		templateclient:     templateClient.Template(),
+		lister:             informer.Lister(),
+		hasSynced:          informer.Informer().HasSynced,
+		templateNamespaces: templateNamespaces,
+		ready:              make(chan struct{}),
+	}
+
+	// TODO this is an intermediate state. Once we're out of tree, there won't be a ready
+	// for now, this skips the hassynced on the lister since we'll be managing that as a poststarthook
+	close(b.ready)
+
+	return b, nil
+}
+
 // MakeReady actually makes the broker functional
 func (b *Broker) MakeReady() error {
 	select {