Merge pull request kubernetes#53831 from gnufied/fix-multiattach-error-flake

Automatic merge from submit-queue (batch tested with PRs 51840, 53542, 53857, 53831, 53702). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Fix possible flake in multiattach unit test

It is possible that by the time we check for multiattach
error on node, the reconciler loop may not have processed second
volume and hence we are going to retry for multiattach error
on node before giving up and marking the test as failed.

Fixes openshift/origin#16836

Author: Kubernetes Submit Queue

pkg/controller/volume/attachdetach/reconciler/reconciler_test.go

@@ -456,15 +456,9 @@ func Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteOnce(t *testing.
 	nodesForVolume := asw.GetNodesForVolume(generatedVolumeName)
 
 	// check if multiattach is marked
-	// at least one volume should be marked with multiattach error
+	// at least one volume+node should be marked with multiattach error
 	nodeAttachedTo := nodesForVolume[0]
-	for _, volumeToAttach := range dsw.GetVolumesToAttach() {
-		if volumeToAttach.NodeName != nodeAttachedTo {
-			if !volumeToAttach.MultiAttachErrorReported {
-				t.Fatalf("Expected volume %q on node %q to have multiattach error", volumeToAttach.VolumeName, volumeToAttach.NodeName)
-			}
-		}
-	}
+	waitForMultiAttachErrorOnNode(t, nodeAttachedTo, dsw)
 
 	// Act
 	podToDelete := ""
@@ -495,6 +489,28 @@ func Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteOnce(t *testing.
 	waitForTotalAttachCallCount(t, 2 /* expectedAttachCallCount */, fakePlugin)
 }
 
+func waitForMultiAttachErrorOnNode(
+	t *testing.T,
+	attachedNode k8stypes.NodeName,
+	dsow cache.DesiredStateOfWorld) {
+	multAttachCheckFunc := func() (bool, error) {
+		for _, volumeToAttach := range dsow.GetVolumesToAttach() {
+			if volumeToAttach.NodeName != attachedNode {
+				if volumeToAttach.MultiAttachErrorReported {
+					return true, nil
+				}
+			}
+		}
+		t.Logf("Warning: MultiAttach error not yet set on Node. Will retry.")
+		return false, nil
+	}
+
+	err := retryWithExponentialBackOff(100*time.Millisecond, multAttachCheckFunc)
+	if err != nil {
+		t.Fatalf("Timed out waiting for MultiAttach Error to be set on non-attached node")
+	}
+}
+
 func waitForNewAttacherCallCount(
 	t *testing.T,
 	expectedCallCount int,