Skip to content

Commit f6e03a5

Browse files
haibchenhaibchen
committed
YARN-6705 Add separate NM preemption thresholds for cpu and memory (Haibo Chen)
1 parent 5baae1b commit f6e03a5

File tree

3 files changed

+85
-22
lines changed

3 files changed

+85
-22
lines changed

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1643,10 +1643,33 @@ public static boolean isAclEnabled(Configuration conf) {
16431643
public static final String NM_OVERALLOCATION_MEMORY_UTILIZATION_THRESHOLD =
16441644
NM_PREFIX + "overallocation.memory-utilization-threshold";
16451645

1646-
public static final String NM_OVERALLOCATION_PREEMPTION_THRESHOLD =
1647-
NM_PREFIX + "overallocation.preemption-threshold";
1648-
public static final float DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD
1649-
= 0.96f;
1646+
/**
1647+
* The CPU utilization threshold, if went beyond for a few times in a row,
1648+
* OPPORTUNISTIC containers started due to overallocation should start
1649+
* getting preempted.
1650+
*/
1651+
public static final String NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD =
1652+
NM_PREFIX + "overallocation.preemption-threshold.cpu";
1653+
public static final float
1654+
DEFAULT_NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD = 0.99f;
1655+
1656+
/**
1657+
* The number of times that CPU utilization must go over the CPU preemption
1658+
* threshold consecutively before preemption starts to kick in.
1659+
*/
1660+
public static final String NM_OVERALLOCATION_PREEMPTION_CPU_COUNT =
1661+
NM_PREFIX + "overallocation.preemption-threshold-count.cpu";
1662+
public static final int DEFAULT_NM_OVERALLOCATION_PREEMPTION_CPU_COUNT = 4;
1663+
1664+
1665+
/**
1666+
* The memory utilization threshold beyond which OPPORTUNISTIC containers
1667+
* started due to overallocation should start getting preempted.
1668+
*/
1669+
public static final String NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD =
1670+
NM_PREFIX + "overallocation.preemption-threshold.memory";
1671+
public static final float
1672+
DEFAULT_NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD = 0.95f;
16501673

16511674
/**
16521675
* Interval of time the linux container executor should try cleaning up

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1588,11 +1588,37 @@
15881588

15891589
<property>
15901590
<description>When a node is over-allocated to improve utilization by
1591-
running OPPORTUNISTIC containers, this config captures the utilization
1592-
beyond which OPPORTUNISTIC containers should start getting preempted.
1591+
running OPPORTUNISTIC containers, this config captures the CPU
1592+
utilization beyond which OPPORTUNISTIC containers should start getting
1593+
preempted. This is used in combination with
1594+
yarn.nodemanager.overallocation.preemption-threshold-count.cpu, that is,
1595+
only when the CPU utilization goes over this threshold consecutively for
1596+
a few times will preemption kicks in.
15931597
</description>
1594-
<name>yarn.nodemanager.overallocation.preemption-threshold</name>
1595-
<value>0.96</value>
1598+
<name>yarn.nodemanager.overallocation.preemption-threshold.cpu</name>
1599+
<value>0.99</value>
1600+
</property>
1601+
1602+
<property>
1603+
<description>When a node is over-allocated to improve utilization by
1604+
running OPPORTUNISTIC containers, this config captures the number of
1605+
times that CPU utilization has to go above
1606+
${yarn.nodemanager.overallocation.preemption-threshold.cpu}
1607+
consecutively for NM to start preempting OPPORTUNISTIC containers
1608+
started due to overallocation.
1609+
</description>
1610+
<name>yarn.nodemanager.overallocation.preemption-threshold-count.cpu</name>
1611+
<value>4</value>
1612+
</property>
1613+
1614+
<property>
1615+
<description>When a node is over-allocated to improve utilization by
1616+
running OPPORTUNISTIC containers, this config captures the CPU
1617+
utilization beyond which OPPORTUNISTIC containers should start getting
1618+
preempted.
1619+
</description>
1620+
<name>yarn.nodemanager.overallocation.preemption-threshold.memory</name>
1621+
<value>0.95</value>
15961622
</property>
15971623

15981624
<property>

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java

Lines changed: 28 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ public enum ContainerMetric {
102102
private ResourceUtilization containersUtilization;
103103

104104
private ResourceThresholds overAllocationPreemptionThresholds;
105+
private int overAlloctionPreemptionCpuCount = -1;
105106

106107
private volatile boolean stopped = false;
107108

@@ -236,7 +237,7 @@ private void initializeOverAllocation(Configuration conf) {
236237
YarnConfiguration.MAX_NM_OVERALLOCATION_THRESHOLD);
237238
if (overAllocationMemoryUtilizationThreshold <= 0) {
238239
LOG.info("NodeManager oversubscription is disabled because the memory " +
239-
"utilization threshold is no larger than zero.");
240+
"overallocation threshold is no larger than zero.");
240241
return;
241242
}
242243

@@ -248,36 +249,49 @@ private void initializeOverAllocation(Configuration conf) {
248249
YarnConfiguration.MAX_NM_OVERALLOCATION_THRESHOLD);
249250
if (overAllocationCpuUtilizationThreshold <= 0) {
250251
LOG.info("NodeManager oversubscription is disabled because the CPU " +
251-
"utilization threshold is no larger than zero.");
252+
"overallocation threshold is no larger than zero.");
252253
return;
253254
}
254255

255-
float preemptionThreshold = conf.getFloat(
256-
YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_THRESHOLD,
257-
YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD);
258-
if (preemptionThreshold <= overAllocationCpuUtilizationThreshold) {
259-
LOG.info("NodeManager oversubscription is disabled because preemption" +
260-
"threshold is no larger than the cpu utilization threshold.");
256+
float cpuPreemptionThreshold = conf.getFloat(
257+
YarnConfiguration.NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD,
258+
YarnConfiguration.
259+
DEFAULT_NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD);
260+
if (cpuPreemptionThreshold <= overAllocationCpuUtilizationThreshold) {
261+
LOG.info("NodeManager oversubscription is disabled because the cpu " +
262+
" preemption threshold is no larger than the cpu overallocation" +
263+
" threshold.");
261264
return;
262265
}
263-
if (preemptionThreshold <= overAllocationMemoryUtilizationThreshold) {
264-
LOG.info("NodeManager oversubscription is disabled because preemption" +
265-
"threshold is no larger than the memory utilization threshold.");
266+
267+
float memoryPreemptionThreshold = conf.getFloat(
268+
YarnConfiguration.NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD,
269+
YarnConfiguration.
270+
DEFAULT_NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD);
271+
if (memoryPreemptionThreshold <= overAllocationMemoryUtilizationThreshold) {
272+
LOG.info("NodeManager oversubscription is disabled because the memory" +
273+
" preemption threshold is no larger than the memory overallocation" +
274+
" threshold.");
266275
return;
267276
}
268277

278+
this.overAlloctionPreemptionCpuCount = conf.getInt(
279+
YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_CPU_COUNT,
280+
YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_CPU_COUNT);
281+
269282
ResourceThresholds resourceThresholds = ResourceThresholds.newInstance(
270283
overAllocationCpuUtilizationThreshold,
271284
overAllocationMemoryUtilizationThreshold);
272285
((NodeManager.NMContext) context).setOverAllocationInfo(
273286
OverAllocationInfo.newInstance(resourceThresholds));
274-
this.overAllocationPreemptionThresholds =
275-
ResourceThresholds.newInstance(preemptionThreshold);
287+
this.overAllocationPreemptionThresholds = ResourceThresholds.newInstance(
288+
cpuPreemptionThreshold, memoryPreemptionThreshold);
276289

277290
LOG.info("NodeManager oversubscription enabled with overallocation " +
278291
"thresholds (memory:" + overAllocationMemoryUtilizationThreshold +
279292
", CPU:" + overAllocationCpuUtilizationThreshold + ") and preemption" +
280-
" threshold: " + preemptionThreshold);
293+
" threshold (memory:" + memoryPreemptionThreshold + ", CPU:" +
294+
cpuPreemptionThreshold + ")");
281295
}
282296

283297
private boolean isResourceCalculatorAvailable() {

0 commit comments

Comments
 (0)