Skip to content

Commit 8f78af1

Browse files
anujmodi2021anujmodi2021
authored
HADOOP-19580. [ABFS][BugFix] IsNonEmptyDirectory Check should loop on listing using updated continuation token (#7716)
Contributed by Anuj Modi Reviewed by Sneha Vijayarajan
1 parent 58fbe4a commit 8f78af1

File tree

3 files changed

+76
-12
lines changed

3 files changed

+76
-12
lines changed

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2037,7 +2037,7 @@ public boolean isNonEmptyDirectory(String path,
20372037
List<FileStatus> fileStatusList = new ArrayList<>();
20382038
// We need to loop on continuation token until we get an entry or continuation token becomes null.
20392039
do {
2040-
ListResponseData listResponseData = listPath(path, false, 1, null, tracingContext, null);
2040+
ListResponseData listResponseData = listPath(path, false, 1, continuationToken, tracingContext, null);
20412041
fileStatusList.addAll(listResponseData.getFileStatusList());
20422042
continuationToken = listResponseData.getContinuationToken();
20432043
} while (StringUtils.isNotEmpty(continuationToken) && fileStatusList.isEmpty());

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -391,9 +391,9 @@ public void testEmptyListingInSubsequentCall() throws IOException {
391391
true, 2, 0);
392392
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, true, EMPTY_STRING,
393393
false, 2, 1);
394-
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, true, TEST_CONTINUATION_TOKEN,
394+
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN + 1, true, TEST_CONTINUATION_TOKEN + 2,
395395
true, 3, 0);
396-
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, true, TEST_CONTINUATION_TOKEN,
396+
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN + 1, true, TEST_CONTINUATION_TOKEN + 2,
397397
false, 3, 1);
398398

399399
testEmptyListingInSubsequentCallInternal(EMPTY_STRING, false, EMPTY_STRING,
@@ -409,9 +409,9 @@ public void testEmptyListingInSubsequentCall() throws IOException {
409409
true, 2, 1);
410410
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, false, EMPTY_STRING,
411411
false, 2, 2);
412-
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, false, TEST_CONTINUATION_TOKEN,
412+
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN + 1, false, TEST_CONTINUATION_TOKEN + 2,
413413
true, 3, 1);
414-
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN, false, TEST_CONTINUATION_TOKEN,
414+
testEmptyListingInSubsequentCallInternal(TEST_CONTINUATION_TOKEN + 1, false, TEST_CONTINUATION_TOKEN + 2,
415415
false, 3, 2);
416416
}
417417

@@ -453,9 +453,23 @@ private void testEmptyListingInSubsequentCallInternal(String firstCT,
453453
listResponseData3.setFileStatusList(new ArrayList<>());
454454
listResponseData3.setOp(Mockito.mock(AbfsRestOperation.class));
455455

456-
Mockito.doReturn(listResponseData1).doReturn(listResponseData2).doReturn(listResponseData3)
457-
.when(spiedClient).listPath(eq("/testPath"), eq(false), eq(1),
458-
any(), any(), any());
456+
final int[] itr = new int[1];
457+
final String[] continuationTokenUsed = new String[3];
458+
459+
Mockito.doAnswer(invocationOnMock -> {
460+
if (itr[0] == 0) {
461+
itr[0]++;
462+
continuationTokenUsed[0] = invocationOnMock.getArgument(3);
463+
return listResponseData1;
464+
} else if (itr[0] == 1) {
465+
itr[0]++;
466+
continuationTokenUsed[1] = invocationOnMock.getArgument(3);
467+
return listResponseData2;
468+
}
469+
continuationTokenUsed[2] = invocationOnMock.getArgument(3);
470+
return listResponseData3;
471+
}).when(spiedClient).listPath(eq("/testPath"), eq(false), eq(1),
472+
any(), any(TracingContext.class), any());
459473

460474
FileStatus[] list = spiedFs.listStatus(new Path("/testPath"));
461475

@@ -473,6 +487,22 @@ private void testEmptyListingInSubsequentCallInternal(String firstCT,
473487
Mockito.verify(spiedClient, times(0))
474488
.getPathStatus(eq("/testPath"), any(), eq(null), eq(false));
475489
}
490+
491+
Assertions.assertThat(continuationTokenUsed[0])
492+
.describedAs("First continuation token used is not as expected")
493+
.isNull();
494+
495+
if (expectedInvocations > 1) {
496+
Assertions.assertThat(continuationTokenUsed[1])
497+
.describedAs("Second continuation token used is not as expected")
498+
.isEqualTo(firstCT);
499+
}
500+
501+
if (expectedInvocations > 2) {
502+
Assertions.assertThat(continuationTokenUsed[2])
503+
.describedAs("Third continuation token used is not as expected")
504+
.isEqualTo(secondCT);
505+
}
476506
}
477507

478508
/**

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java

Lines changed: 38 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -745,18 +745,18 @@ public void testIsNonEmptyDirectory() throws IOException {
745745
true, 2, false);
746746
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, true, EMPTY_STRING,
747747
false, 2, true);
748-
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, true, TEST_CONTINUATION_TOKEN,
748+
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN + 1, true, TEST_CONTINUATION_TOKEN + 2,
749749
true, 3, false);
750-
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, true, TEST_CONTINUATION_TOKEN,
750+
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN + 1, true, TEST_CONTINUATION_TOKEN + 2,
751751
false, 2, true);
752752

753753
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, false, EMPTY_STRING,
754754
true, 1, true);
755755
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, false, EMPTY_STRING,
756756
false, 1, true);
757-
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, false, TEST_CONTINUATION_TOKEN,
757+
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN + 1, false, TEST_CONTINUATION_TOKEN + 2,
758758
true, 1, true);
759-
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN, false, TEST_CONTINUATION_TOKEN,
759+
testIsNonEmptyDirectoryInternal(TEST_CONTINUATION_TOKEN + 1, false, TEST_CONTINUATION_TOKEN + 2,
760760
false, 1, true);
761761
}
762762

@@ -801,11 +801,45 @@ private void testIsNonEmptyDirectoryInternal(String firstCT,
801801
.when(spiedClient).listPath(eq("/testPath"), eq(false), eq(1),
802802
any(), any(), any());
803803

804+
final int[] itr = new int[1];
805+
final String[] continuationTokenUsed = new String[3];
806+
807+
Mockito.doAnswer(invocationOnMock -> {
808+
if (itr[0] == 0) {
809+
itr[0]++;
810+
continuationTokenUsed[0] = invocationOnMock.getArgument(3);
811+
return listResponseData1;
812+
} else if (itr[0] == 1) {
813+
itr[0]++;
814+
continuationTokenUsed[1] = invocationOnMock.getArgument(3);
815+
return listResponseData2;
816+
}
817+
continuationTokenUsed[2] = invocationOnMock.getArgument(3);
818+
return listResponseData3;
819+
}).when(spiedClient).listPath(eq("/testPath"), eq(false), eq(1),
820+
any(), any(TracingContext.class), any());
821+
804822
Assertions.assertThat(spiedClient.isNonEmptyDirectory("/testPath",
805823
Mockito.mock(TracingContext.class)))
806824
.describedAs("isNonEmptyDirectory in client giving unexpected results")
807825
.isEqualTo(isNonEmpty);
808826

827+
Assertions.assertThat(continuationTokenUsed[0])
828+
.describedAs("First continuation token used is not as expected")
829+
.isNull();
830+
831+
if (expectedInvocations > 1) {
832+
Assertions.assertThat(continuationTokenUsed[1])
833+
.describedAs("Second continuation token used is not as expected")
834+
.isEqualTo(firstCT);
835+
}
836+
837+
if (expectedInvocations > 2) {
838+
Assertions.assertThat(continuationTokenUsed[2])
839+
.describedAs("Third continuation token used is not as expected")
840+
.isEqualTo(secondCT);
841+
}
842+
809843
Mockito.verify(spiedClient, times(expectedInvocations))
810844
.listPath(eq("/testPath"), eq(false), eq(1),
811845
any(), any(TracingContext.class), any());

0 commit comments

Comments
 (0)