Enable leader election on endpoints for controllers

Support the new upstream module for leader election via a new config
field and command line flag (--lock-service-name). If specified, the new
style election will be used. The legacy etcd election (triggered by
controllerTTL > 0) will wait to verify no endpoint object exists before
competing for the etcd lease, and will step down if it detects the
endpoint object is created.

With these changes, the controllers can now be run as static pods on the
masters and talk only to the API. This will allow them to appear in the
api and be scraped by prometheus.

Author: smarterclayton

contrib/completions/bash/openshift

@@ -35173,6 +35173,8 @@ _openshift_start_master_controllers()
     local_nonpersistent_flags+=("--config=")
     flags+=("--listen=")
     local_nonpersistent_flags+=("--listen=")
+    flags+=("--lock-service-name=")
+    local_nonpersistent_flags+=("--lock-service-name=")
     flags+=("--azure-container-registry-config=")
     flags+=("--google-json-key=")
     flags+=("--log-flush-frequency=")

contrib/completions/zsh/openshift

@@ -35322,6 +35322,8 @@ _openshift_start_master_controllers()
     local_nonpersistent_flags+=("--config=")
     flags+=("--listen=")
     local_nonpersistent_flags+=("--listen=")
+    flags+=("--lock-service-name=")
+    local_nonpersistent_flags+=("--lock-service-name=")
     flags+=("--azure-container-registry-config=")
     flags+=("--google-json-key=")
     flags+=("--log-flush-frequency=")

contrib/kubernetes/controllers.yaml

@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: controllers
+  labels:
+    master.openshift.io/controllers: 'true'
+spec:
+  containers:
+  - name: controller
+    image: openshift/origin:latest
+    args:
+    - start
+    - master
+    - controllers
+    - --listen=0.0.0.0:8444
+    - --config=/etc/origin/master/master-config.yaml
+    volumeMounts:
+    - name: config
+      mountPath: /etc/origin/master
+    ports:
+    - containerPort: 8444
+      name: https
+  volumes:
+  - hostPath:
+      path: /data/src/github.com/openshift/origin/openshift.local.test/master
+    name: config

pkg/cmd/server/api/serialization_test.go

@@ -43,6 +43,14 @@ func fuzzInternalObject(t *testing.T, forVersion schema.GroupVersion, item runti
 			if len(obj.Controllers) == 0 {
 				obj.Controllers = configapi.ControllersAll
 			}
+			if election := obj.ControllerConfig.Election; election != nil {
+				if len(election.LockNamespace) == 0 {
+					election.LockNamespace = "kube-system"
+				}
+				if len(election.LockResource.Group) == 0 && len(election.LockResource.Resource) == 0 {
+					election.LockResource.Resource = "endpoints"
+				}
+			}
 			if obj.ServingInfo.RequestTimeoutSeconds == 0 {
 				obj.ServingInfo.RequestTimeoutSeconds = 60 * 60
 			}

pkg/cmd/server/api/types.go

@@ -334,12 +334,14 @@ type MasterConfig struct {
 	Controllers string
 	// PauseControllers instructs the master to not automatically start controllers, but instead
 	// to wait until a notification to the server is received before launching them.
-	// TODO: will be disabled in function for 1.1.
+	// Deprecated: Will be removed in 3.7.
 	PauseControllers bool
-	// ControllerLeaseTTL enables controller election, instructing the master to attempt to acquire
-	// a lease before controllers start and renewing it within a number of seconds defined by this value.
-	// Setting this value non-negative forces pauseControllers=true. This value defaults off (0, or
+	// ControllerLeaseTTL enables controller election against etcd, instructing the master to attempt to
+	// acquire a lease before controllers start and renewing it within a number of seconds defined by this
+	// value. Setting this value non-negative forces pauseControllers=true. This value defaults off (0, or
 	// omitted) and controller election can be disabled with -1.
+	// Deprecated: use controllerConfig.lockServiceName to force leader election via config, and the
+	//   appropriate leader election flags in controllerArguments. Will be removed in 3.9.
 	ControllerLeaseTTL int
 	// TODO: the next field added to controllers must be added to a new controllers struct
 
@@ -1393,11 +1395,37 @@ type AdmissionConfig struct {
 
 // ControllerConfig holds configuration values for controllers
 type ControllerConfig struct {
+	// Election defines the configuration for electing a controller instance to make changes to
+	// the cluster. If unspecified, the ControllerTTL value is checked to determine whether the
+	// legacy direct etcd election code will be used.
+	Election *ControllerElectionConfig
 	// ServiceServingCert holds configuration for service serving cert signer which creates cert/key pairs for
 	// pods fulfilling a service to serve with.
 	ServiceServingCert ServiceServingCert
 }
 
+// ControllerElectionConfig contains configuration values for deciding how a controller
+// will be elected to act as leader.
+type ControllerElectionConfig struct {
+	// LockName is the resource name used to act as the lock for determining which controller
+	// instance should lead.
+	LockName string
+	// LockNamespace is the resource namespace used to act as the lock for determining which
+	// controller instance should lead. It defaults to "kube-system"
+	LockNamespace string
+	// LockResource is the group and resource name to use to coordinate for the controller lock.
+	// If unset, defaults to "endpoints".
+	LockResource GroupResource
+}
+
+// GroupResource points to a resource by its name and API group.
+type GroupResource struct {
+	// Group is the name of an API group
+	Group string
+	// Resource is the name of a resource.
+	Resource string
+}
+
 // ServiceServingCert holds configuration for service serving cert signer which creates cert/key pairs for
 // pods fulfilling a service to serve with.
 type ServiceServingCert struct {

pkg/cmd/server/api/v1/conversions.go

@@ -20,6 +20,14 @@ func addDefaultingFuncs(scheme *runtime.Scheme) error {
 			if len(obj.Controllers) == 0 {
 				obj.Controllers = ControllersAll
 			}
+			if election := obj.ControllerConfig.Election; election != nil {
+				if len(election.LockNamespace) == 0 {
+					election.LockNamespace = "kube-system"
+				}
+				if len(election.LockResource.Group) == 0 && len(election.LockResource.Resource) == 0 {
+					election.LockResource.Resource = "endpoints"
+				}
+			}
 			if obj.ServingInfo.RequestTimeoutSeconds == 0 {
 				obj.ServingInfo.RequestTimeoutSeconds = 60 * 60
 			}

pkg/cmd/server/api/v1/swagger_doc.go

@@ -133,13 +133,25 @@ func (ClientConnectionOverrides) SwaggerDoc() map[string]string {
 
 var map_ControllerConfig = map[string]string{
 	"":                   "ControllerConfig holds configuration values for controllers",
+	"election":           "Election defines the configuration for electing a controller instance to make changes to the cluster. If unspecified, the ControllerTTL value is checked to determine whether the legacy direct etcd election code will be used.",
 	"serviceServingCert": "ServiceServingCert holds configuration for service serving cert signer which creates cert/key pairs for pods fulfilling a service to serve with.",
 }
 
 func (ControllerConfig) SwaggerDoc() map[string]string {
 	return map_ControllerConfig
 }
 
+var map_ControllerElectionConfig = map[string]string{
+	"":              "ControllerElectionConfig contains configuration values for deciding how a controller will be elected to act as leader.",
+	"lockName":      "LockName is the resource name used to act as the lock for determining which controller instance should lead.",
+	"lockNamespace": "LockNamespace is the resource namespace used to act as the lock for determining which controller instance should lead. It defaults to \"kube-system\"",
+	"lockResource":  "LockResource is the group and resource name to use to coordinate for the controller lock. If unset, defaults to \"Endpoints\".",
+}
+
+func (ControllerElectionConfig) SwaggerDoc() map[string]string {
+	return map_ControllerElectionConfig
+}
+
 var map_DNSConfig = map[string]string{
 	"":                      "DNSConfig holds the necessary configuration options for DNS",
 	"bindAddress":           "BindAddress is the ip:port to serve DNS on",
@@ -259,6 +271,16 @@ func (GrantConfig) SwaggerDoc() map[string]string {
 	return map_GrantConfig
 }
 
+var map_GroupResource = map[string]string{
+	"":         "GroupResource points to a resource by its name and API group.",
+	"group":    "Group is the name of an API group",
+	"resource": "Resource is the name of a resource.",
+}
+
+func (GroupResource) SwaggerDoc() map[string]string {
+	return map_GroupResource
+}
+
 var map_HTPasswdPasswordIdentityProvider = map[string]string{
 	"":     "HTPasswdPasswordIdentityProvider provides identities for users authenticating using htpasswd credentials",
 	"file": "File is a reference to your htpasswd file",
@@ -463,8 +485,8 @@ var map_MasterConfig = map[string]string{
 	"apiLevels":                   "APILevels is a list of API levels that should be enabled on startup: v1 as examples",
 	"masterPublicURL":             "MasterPublicURL is how clients can access the OpenShift API server",
 	"controllers":                 "Controllers is a list of the controllers that should be started. If set to \"none\", no controllers will start automatically. The default value is \"*\" which will start all controllers. When using \"*\", you may exclude controllers by prepending a \"-\" in front of their name. No other values are recognized at this time.",
-	"pauseControllers":            "PauseControllers instructs the master to not automatically start controllers, but instead to wait until a notification to the server is received before launching them.",
-	"controllerLeaseTTL":          "ControllerLeaseTTL enables controller election, instructing the master to attempt to acquire a lease before controllers start and renewing it within a number of seconds defined by this value. Setting this value non-negative forces pauseControllers=true. This value defaults off (0, or omitted) and controller election can be disabled with -1.",
+	"pauseControllers":            "PauseControllers instructs the master to not automatically start controllers, but instead to wait until a notification to the server is received before launching them. This field is ignored if controllerConfig.lockServiceName is specified. Deprecated: Will be removed in 3.7.",
+	"controllerLeaseTTL":          "ControllerLeaseTTL enables controller election against etcd, instructing the master to attempt to acquire a lease before controllers start and renewing it within a number of seconds defined by this value. Setting this value non-negative forces pauseControllers=true. This value defaults off (0, or omitted) and controller election can be disabled with -1. This field is ignored if controllerConfig.lockServiceName is specified. Deprecated: use controllerConfig.lockServiceName to force leader election via config, and the\n  appropriate leader election flags in controllerArguments. Will be removed in 3.9.",
 	"admissionConfig":             "AdmissionConfig contains admission control plugin configuration.",
 	"controllerConfig":            "ControllerConfig holds configuration values for controllers",
 	"disabledFeatures":            "DisabledFeatures is a list of features that should not be started.  We omitempty here because its very unlikely that anyone will want to manually disable features and we don't want to encourage it.",

pkg/cmd/server/api/v1/types.go

@@ -197,12 +197,17 @@ type MasterConfig struct {
 	// values are recognized at this time.
 	Controllers string `json:"controllers"`
 	// PauseControllers instructs the master to not automatically start controllers, but instead
-	// to wait until a notification to the server is received before launching them.
+	// to wait until a notification to the server is received before launching them. This field is
+	// ignored if controllerConfig.lockServiceName is specified.
+	// Deprecated: Will be removed in 3.7.
 	PauseControllers bool `json:"pauseControllers"`
-	// ControllerLeaseTTL enables controller election, instructing the master to attempt to acquire
-	// a lease before controllers start and renewing it within a number of seconds defined by this value.
-	// Setting this value non-negative forces pauseControllers=true. This value defaults off (0, or
-	// omitted) and controller election can be disabled with -1.
+	// ControllerLeaseTTL enables controller election against etcd, instructing the master to attempt to
+	// acquire a lease before controllers start and renewing it within a number of seconds defined by this
+	// value. Setting this value non-negative forces pauseControllers=true. This value defaults off (0, or
+	// omitted) and controller election can be disabled with -1. This field is ignored if
+	// controllerConfig.lockServiceName is specified.
+	// Deprecated: use controllerConfig.lockServiceName to force leader election via config, and the
+	//   appropriate leader election flags in controllerArguments. Will be removed in 3.9.
 	ControllerLeaseTTL int `json:"controllerLeaseTTL"`
 
 	// AdmissionConfig contains admission control plugin configuration.
@@ -1327,11 +1332,37 @@ type AdmissionConfig struct {
 
 // ControllerConfig holds configuration values for controllers
 type ControllerConfig struct {
+	// Election defines the configuration for electing a controller instance to make changes to
+	// the cluster. If unspecified, the ControllerTTL value is checked to determine whether the
+	// legacy direct etcd election code will be used.
+	Election *ControllerElectionConfig `json:"election"`
 	// ServiceServingCert holds configuration for service serving cert signer which creates cert/key pairs for
 	// pods fulfilling a service to serve with.
 	ServiceServingCert ServiceServingCert `json:"serviceServingCert"`
 }
 
+// ControllerElectionConfig contains configuration values for deciding how a controller
+// will be elected to act as leader.
+type ControllerElectionConfig struct {
+	// LockName is the resource name used to act as the lock for determining which controller
+	// instance should lead.
+	LockName string `json:"lockName"`
+	// LockNamespace is the resource namespace used to act as the lock for determining which
+	// controller instance should lead. It defaults to "kube-system"
+	LockNamespace string `json:"lockNamespace"`
+	// LockResource is the group and resource name to use to coordinate for the controller lock.
+	// If unset, defaults to "Endpoints".
+	LockResource GroupResource `json:"lockResource"`
+}
+
+// GroupResource points to a resource by its name and API group.
+type GroupResource struct {
+	// Group is the name of an API group
+	Group string `json:"group"`
+	// Resource is the name of a resource.
+	Resource string `json:"resource"`
+}
+
 // ServiceServingCert holds configuration for service serving cert signer which creates cert/key pairs for
 // pods fulfilling a service to serve with.
 type ServiceServingCert struct {

pkg/cmd/server/api/v1/types_test.go

@@ -118,6 +118,7 @@ auditConfig:
 authConfig:
   requestHeader: null
 controllerConfig:
+  election: null
   serviceServingCert:
     signer: null
 controllerLeaseTTL: 0

pkg/cmd/server/api/validation/master.go

@@ -249,6 +249,23 @@ func ValidateAuditConfig(config api.AuditConfig, fldPath *field.Path) Validation
 func ValidateControllerConfig(config api.ControllerConfig, fldPath *field.Path) ValidationResults {
 	validationResults := ValidationResults{}
 
+	if election := config.Election; election != nil {
+		if len(election.LockName) == 0 {
+			validationResults.AddErrors(field.Invalid(fldPath.Child("election", "lockName"), election.LockName, "may not be empty"))
+		}
+		for _, msg := range kvalidation.ValidateServiceName(election.LockName, false) {
+			validationResults.AddErrors(field.Invalid(fldPath.Child("election", "lockName"), election.LockName, msg))
+		}
+		if len(election.LockNamespace) == 0 {
+			validationResults.AddErrors(field.Invalid(fldPath.Child("election", "lockNamespace"), election.LockNamespace, "may not be empty"))
+		}
+		for _, msg := range kvalidation.ValidateNamespaceName(election.LockNamespace, false) {
+			validationResults.AddErrors(field.Invalid(fldPath.Child("election", "lockNamespace"), election.LockNamespace, msg))
+		}
+		if len(election.LockResource.Resource) == 0 {
+			validationResults.AddErrors(field.Invalid(fldPath.Child("election", "lockResource", "resource"), election.LockResource.Resource, "may not be empty"))
+		}
+	}
 	if config.ServiceServingCert.Signer != nil {
 		validationResults.AddErrors(ValidateCertInfo(*config.ServiceServingCert.Signer, true, fldPath.Child("serviceServingCert.signer"))...)
 	}