Bug 1596440 - surface OOMKilled pod to build

Author: Jan Wozniak

pkg/build/apis/build/types.go

@@ -514,6 +514,9 @@ const (
 	// range of build failures.
 	StatusReasonGenericBuildFailed StatusReason = "GenericBuildFailed"
 
+	// StatusReasonOutOfMemoryKilled indicates that the build pod was killed for its memory consumption
+	StatusReasonOutOfMemoryKilled StatusReason = "OutOfMemoryKilled"
+
 	// StatusCannotRetrieveServiceAccount is the reason associated with a failure
 	// to look up the service account associated with the BuildConfig.
 	StatusReasonCannotRetrieveServiceAccount StatusReason = "CannotRetrieveServiceAccount"
@@ -540,6 +543,7 @@ const (
 	StatusMessageNoBuildContainerStatus          = "The pod for this build has no container statuses indicating success or failure."
 	StatusMessageFailedContainer                 = "The pod for this build has at least one container with a non-zero exit status."
 	StatusMessageGenericBuildFailed              = "Generic Build failure - check logs for details."
+	StatusMessageOutOfMemoryKilled               = "The build pod was killed due to an out of memory condition."
 	StatusMessageUnresolvableEnvironmentVariable = "Unable to resolve build environment variable reference."
 	StatusMessageCannotRetrieveServiceAccount    = "Unable to look up the service account secrets for this build."
 )

pkg/build/controller/build/build_controller.go

@@ -1006,6 +1006,8 @@ func (bc *BuildController) handleActiveBuild(build *buildapi.Build, pod *v1.Pod)
 			// soon be deleted. The build should be transitioned to the Error phase.
 			if pod.DeletionTimestamp != nil {
 				update = transitionToPhase(buildapi.BuildPhaseError, buildapi.StatusReasonBuildPodDeleted, buildapi.StatusMessageBuildPodDeleted)
+			} else if isOOMKilled(pod) {
+				update = transitionToPhase(buildapi.BuildPhaseFailed, buildapi.StatusReasonOutOfMemoryKilled, buildapi.StatusMessageOutOfMemoryKilled)
 			} else {
 				update = transitionToPhase(buildapi.BuildPhaseFailed, buildapi.StatusReasonGenericBuildFailed, buildapi.StatusMessageGenericBuildFailed)
 			}
@@ -1014,6 +1016,25 @@ func (bc *BuildController) handleActiveBuild(build *buildapi.Build, pod *v1.Pod)
 	return update, nil
 }
 
+func isOOMKilled(pod *v1.Pod) bool {
+	if pod.Status.Reason == "OOMKilled" {
+		return true
+	}
+	for _, c := range pod.Status.InitContainerStatuses {
+		terminated := c.State.Terminated
+		if terminated != nil && terminated.Reason == "OOMKilled" {
+			return true
+		}
+	}
+	for _, c := range pod.Status.ContainerStatuses {
+		terminated := c.State.Terminated
+		if terminated != nil && terminated.Reason == "OOMKilled" {
+			return true
+		}
+	}
+	return false
+}
+
 // handleCompletedBuild will only be called on builds that are already in a terminal phase.  It is used to setup the
 // completion timestamp and failure logsnippet as needed.
 func (bc *BuildController) handleCompletedBuild(build *buildapi.Build, pod *v1.Pod) (*buildUpdate, error) {