apps: deployment config stuck in the new state should respect timeoutSecods

Author: mfojtik

pkg/apps/apis/apps/types.go

@@ -113,6 +113,7 @@ const (
 	DeploymentCancelledNewerDeploymentExists  = "newer deployment was found running"
 	DeploymentFailedUnrelatedDeploymentExists = "unrelated pod with the same name as this deployment is already running"
 	DeploymentFailedDeployerPodNoLongerExists = "deployer pod no longer exists"
+	DeploymentFailedUnableToCreateDeployerPod = "unable to create deployer pod"
 )
 
 // DeploymentStatus describes the possible states a deployment can be in.

pkg/apps/controller/deployer/deployer_controller.go

@@ -127,6 +127,19 @@ func (c *DeploymentController) handle(deployment *v1.ReplicationController, will
 			}
 			break
 		}
+		// In case the deployment is stuck in "new" state because we fail to create
+		// deployer pod (quota, etc..) we should respect the timeoutSeconds in the
+		// config strategy and transition the rollout to failed instead of waiting for
+		// the deployment pod forever.
+		config, err := deployutil.DecodeDeploymentConfig(deployment, c.codec)
+		if err != nil {
+			return err
+		}
+		if deployutil.RolloutExceededTimeoutSeconds(config, deployment) {
+			nextStatus = deployapi.DeploymentStatusFailed
+			updatedAnnotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentFailedUnableToCreateDeployerPod
+			break
+		}
 
 		switch {
 		case kerrors.IsNotFound(deployerErr):

pkg/apps/controller/deploymentconfig/deploymentconfig_controller.go

@@ -128,6 +128,7 @@ func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig)
 			return err
 		}
 	}
+
 	// Process triggers and start an initial rollouts
 	configCopy, err := deployutil.DeploymentConfigDeepCopy(config)
 	if err != nil {

pkg/apps/util/util.go

@@ -652,6 +652,12 @@ func IsTerminatedDeployment(deployment runtime.Object) bool {
 	return IsCompleteDeployment(deployment) || IsFailedDeployment(deployment)
 }
 
+// IsNewDeployment returns true if the passed deployment is in new state.
+func IsNewDeployment(deployment runtime.Object) bool {
+	current := DeploymentStatusFor(deployment)
+	return current == deployapi.DeploymentStatusNew
+}
+
 // IsCompleteDeployment returns true if the passed deployment is in state complete.
 func IsCompleteDeployment(deployment runtime.Object) bool {
 	current := DeploymentStatusFor(deployment)
@@ -782,6 +788,23 @@ func DeploymentsForCleanup(configuration *deployapi.DeploymentConfig, deployment
 	return relevantDeployments
 }
 
+func RolloutExceededTimeoutSeconds(config *deployapi.DeploymentConfig, latestRC *v1.ReplicationController) bool {
+	var timeoutSeconds int64
+	if params := config.Spec.Strategy.RollingParams; params != nil {
+		timeoutSeconds = deployapi.DefaultRollingTimeoutSeconds
+		if params.TimeoutSeconds != nil {
+			timeoutSeconds = *params.TimeoutSeconds
+		}
+	}
+	if params := config.Spec.Strategy.RecreateParams; params != nil {
+		timeoutSeconds = deployapi.DefaultRecreateTimeoutSeconds
+		if params.TimeoutSeconds != nil {
+			timeoutSeconds = *params.TimeoutSeconds
+		}
+	}
+	return int64(time.Since(latestRC.CreationTimestamp.Time)*time.Second) > timeoutSeconds
+}
+
 // WaitForRunningDeployerPod waits a given period of time until the deployer pod
 // for given replication controller is not running.
 func WaitForRunningDeployerPod(podClient kcoreclient.PodsGetter, rc *api.ReplicationController, timeout time.Duration) error {