Skip to content

Commit 0ad9ef8

Browse files
Miklos Szegedihaibchen
Miklos Szegedi
authored and
haibchen
committed
YARN-7491. Make sure AM is not scheduled on an opportunistic container. Contributed by Haibo Chen.
1 parent d6913d5 commit 0ad9ef8

File tree

4 files changed

+130
-18
lines changed
  • hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server

4 files changed

+130
-18
lines changed

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -349,15 +349,28 @@ public static Priority newPriority(int p) {
349349
return priority;
350350
}
351351

352+
public static ResourceRequest newAMResourceRequest(Priority priority,
353+
String hostName, Resource capability) {
354+
return newResourceRequest(priority, hostName, capability, 1,
355+
ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
356+
}
357+
352358
public static ResourceRequest newResourceRequest(Priority priority,
353359
String hostName, Resource capability, int numContainers) {
360+
return newResourceRequest(priority, hostName, capability,
361+
numContainers, ExecutionTypeRequest.newInstance());
362+
}
363+
364+
public static ResourceRequest newResourceRequest(Priority priority,
365+
String hostName, Resource capability, int numContainers,
366+
ExecutionTypeRequest executionTypeRequest) {
354367
ResourceRequest request = recordFactory
355368
.newRecordInstance(ResourceRequest.class);
356369
request.setPriority(priority);
357370
request.setResourceName(hostName);
358371
request.setCapability(capability);
359372
request.setNumContainers(numContainers);
360-
request.setExecutionTypeRequest(ExecutionTypeRequest.newInstance());
373+
request.setExecutionTypeRequest(executionTypeRequest);
361374
return request;
362375
}
363376

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.yarn.server.utils;
20+
21+
import org.apache.hadoop.yarn.api.records.ExecutionType;
22+
import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
23+
import org.apache.hadoop.yarn.api.records.Priority;
24+
import org.apache.hadoop.yarn.api.records.Resource;
25+
import org.apache.hadoop.yarn.api.records.ResourceRequest;
26+
import org.junit.Assert;
27+
import org.junit.Test;
28+
29+
/**
30+
* Unit tests for BuilderUtils.
31+
*/
32+
public class TestBuilderUtils {
33+
@Test
34+
public void testNewAMResourceRequest() {
35+
final Priority priority = Priority.newInstance(0);
36+
final String hostName = "ResourceA";
37+
final Resource capacity = Resource.newInstance(1024, 1);
38+
39+
ResourceRequest resourceRequest =
40+
BuilderUtils.newAMResourceRequest(priority, hostName, capacity);
41+
42+
Assert.assertEquals("unexpected priority", priority,
43+
resourceRequest.getPriority());
44+
Assert.assertEquals("unexpected host name", hostName,
45+
resourceRequest.getResourceName());
46+
Assert.assertEquals("unexpected capacity", capacity,
47+
resourceRequest.getCapability());
48+
Assert.assertEquals("unexpected number of containers", 1,
49+
resourceRequest.getNumContainers());
50+
Assert.assertEquals("unexpected execution type",
51+
ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true),
52+
resourceRequest.getExecutionTypeRequest());
53+
}
54+
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -488,8 +488,8 @@ private List<ResourceRequest> validateAndCreateResourceRequest(
488488
if (amReqs == null || amReqs.isEmpty()) {
489489
if (submissionContext.getResource() != null) {
490490
amReqs = Collections.singletonList(BuilderUtils
491-
.newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
492-
ResourceRequest.ANY, submissionContext.getResource(), 1));
491+
.newAMResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
492+
ResourceRequest.ANY, submissionContext.getResource()));
493493
} else {
494494
throw new InvalidResourceRequestException("Invalid resource request, "
495495
+ "no resources requested");
@@ -520,7 +520,7 @@ private List<ResourceRequest> validateAndCreateResourceRequest(
520520
for (ResourceRequest amReq : amReqs) {
521521
amReq.setCapability(anyReq.getCapability());
522522
amReq.setExecutionTypeRequest(
523-
ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
523+
ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
524524
amReq.setNumContainers(1);
525525
amReq.setPriority(RMAppAttemptImpl.AM_CONTAINER_PRIORITY);
526526
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java

Lines changed: 59 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -691,15 +691,45 @@ protected void setupDispatcher(RMContext rmContext, Configuration conf) {
691691
public void testRMAppSubmitAMContainerResourceRequests() throws Exception {
692692
asContext.setResource(Resources.createResource(1024));
693693
asContext.setAMContainerResourceRequest(
694-
ResourceRequest.newInstance(Priority.newInstance(0),
695-
ResourceRequest.ANY, Resources.createResource(1024), 1, true));
694+
ResourceRequest.newBuilder().priority(Priority.newInstance(0))
695+
.resourceName(ResourceRequest.ANY)
696+
.capability(Resources.createResource(1024))
697+
.numContainers(1)
698+
.relaxLocality(true)
699+
.executionTypeRequest(ExecutionTypeRequest.newInstance(
700+
ExecutionType.GUARANTEED, true))
701+
.build());
696702
List<ResourceRequest> reqs = new ArrayList<>();
697-
reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
698-
ResourceRequest.ANY, Resources.createResource(1025), 1, false));
699-
reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
700-
"/rack", Resources.createResource(1025), 1, false));
701-
reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
702-
"/rack/node", Resources.createResource(1025), 1, true));
703+
reqs.add(
704+
ResourceRequest.newBuilder().priority(Priority.newInstance(0))
705+
.resourceName(ResourceRequest.ANY)
706+
.capability(Resources.createResource(1025))
707+
.numContainers(1)
708+
.relaxLocality(false)
709+
.executionTypeRequest(ExecutionTypeRequest.newInstance(
710+
ExecutionType.GUARANTEED, true))
711+
.build()
712+
);
713+
reqs.add(
714+
ResourceRequest.newBuilder().priority(Priority.newInstance(0))
715+
.resourceName("/rack")
716+
.capability(Resources.createResource(1025))
717+
.numContainers(1)
718+
.relaxLocality(false)
719+
.executionTypeRequest(ExecutionTypeRequest.newInstance(
720+
ExecutionType.GUARANTEED, true))
721+
.build()
722+
);
723+
reqs.add(
724+
ResourceRequest.newBuilder().priority(Priority.newInstance(0))
725+
.resourceName("/rack/node")
726+
.capability(Resources.createResource(1025))
727+
.numContainers(1)
728+
.relaxLocality(true)
729+
.executionTypeRequest(ExecutionTypeRequest.newInstance(
730+
ExecutionType.GUARANTEED, true))
731+
.build()
732+
);
703733
asContext.setAMContainerResourceRequests(cloneResourceRequests(reqs));
704734
// getAMContainerResourceRequest uses the first entry of
705735
// getAMContainerResourceRequests
@@ -720,8 +750,14 @@ public void testRMAppSubmitAMContainerResourceRequest() throws Exception {
720750
asContext.setResource(Resources.createResource(1024));
721751
asContext.setAMContainerResourceRequests(null);
722752
ResourceRequest req =
723-
ResourceRequest.newInstance(Priority.newInstance(0),
724-
ResourceRequest.ANY, Resources.createResource(1025), 1, true);
753+
ResourceRequest.newBuilder().priority(Priority.newInstance(0))
754+
.resourceName(ResourceRequest.ANY)
755+
.capability(Resources.createResource(1025))
756+
.numContainers(1)
757+
.relaxLocality(true)
758+
.executionTypeRequest(ExecutionTypeRequest.newInstance(
759+
ExecutionType.GUARANTEED, true))
760+
.build();
725761
asContext.setAMContainerResourceRequest(cloneResourceRequest(req));
726762
// getAMContainerResourceRequests uses a singleton list of
727763
// getAMContainerResourceRequest
@@ -742,8 +778,16 @@ public void testRMAppSubmitResource() throws Exception {
742778
RMApp app = testRMAppSubmit();
743779
// setResource
744780
Assert.assertEquals(Collections.singletonList(
745-
ResourceRequest.newInstance(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
746-
ResourceRequest.ANY, Resources.createResource(1024), 1, true, "")),
781+
ResourceRequest.newBuilder()
782+
.priority(RMAppAttemptImpl.AM_CONTAINER_PRIORITY)
783+
.resourceName(ResourceRequest.ANY)
784+
.capability(Resources.createResource(1024))
785+
.numContainers(1)
786+
.relaxLocality(true)
787+
.nodeLabelExpression("")
788+
.executionTypeRequest(ExecutionTypeRequest.newInstance(
789+
ExecutionType.GUARANTEED, true))
790+
.build()),
747791
app.getAMResourceRequests());
748792
}
749793

@@ -783,7 +827,7 @@ public void testRMAppSubmitAMContainerResourceRequestsDisagree()
783827
for (ResourceRequest req : reqs) {
784828
req.setCapability(anyReq.getCapability());
785829
req.setExecutionTypeRequest(
786-
ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
830+
ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
787831
req.setNumContainers(1);
788832
req.setPriority(Priority.newInstance(0));
789833
}
@@ -1100,7 +1144,8 @@ private static ResourceRequest cloneResourceRequest(ResourceRequest req) {
11001144
req.getNodeLabelExpression() != null
11011145
? new String(req.getNodeLabelExpression()) : null,
11021146
ExecutionTypeRequest.newInstance(
1103-
req.getExecutionTypeRequest().getExecutionType()));
1147+
req.getExecutionTypeRequest().getExecutionType(),
1148+
req.getExecutionTypeRequest().getEnforceExecutionType()));
11041149
}
11051150

11061151
private static List<ResourceRequest> cloneResourceRequests(

0 commit comments

Comments
 (0)