Skip to content

Commit ceebf3f

Browse files
924060929924060929
authored
[opt](nereids) optimize small sql (#43546)
this pr improve QPS for small sql, by this points: 1. try best use **Visitor** rewriter to traverse plan, instead of **Stack** rewriter 2. fast skip traverse plan tree, if the tree not contains the specify type. for example, if not contains LogicalApply, we will not traverse for the rules of `Subquery unnesting` topic 3. use **RoaringBitmap** to check whether the slot is exist, instead of `Set<Slot>` 4. skip prune column for logicalFilter(logicalOlapScan), or else need more rules to optimize it. for example, logicalProject(logicalFilter(localcalOlapScan)) → ColumnPruning → logicalProject(logicalFilter(logicalProject(localcalOlapScan))) → PushDownFilterThroughProject → logicalProject(logicalProject(logicalFilter(localcalOlapScan))) → MergeProjects → logicalProject(logicalFilter(localcalOlapScan))) 5. use for loop instead of collection's stream api 6. faster process string, like format, split 7. try to process project in one rule as possible, for example, project(project(oneRowRelatoin)) → oneRowRelation 8. fast compute some hashCode, e.g. LogicalOlapScan only use relationId as hash code 9. reuse ConnectContext to generate next exprId without invoke ConnectContext.get() multiple times 10. don't find table multiple times for compute lots of ColumnStatistic which have same table this sql can optimize from 21000 QPS to 32000 QPS ```sql explain select date from test.tbl ``` this sql can optimize from 16000 QPS to 23000 QPS ```sql explain select col1, col2, ..., col16 from test.tbl2 where col1 = 2 and col2 = 'xxx' and col3 >= 1721480971 and col3 <= 1721507345 and col4>=20240720 and col4<=20240721 ; ```
1 parent 80ce5b9 commit ceebf3f

File tree

323 files changed

+5639
-2599
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

323 files changed

+5639
-2599
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -377,7 +377,6 @@ public void setMutableState(String key, Object value) {
377377
@Override
378378
public Expression withChildren(List<Expression> children) {
379379
throw new AnalysisException("could not call withChildren on UdfSignatureSearcher");
380-
381380
}
382381
}
383382
}

‎fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ public List<Tablet> getTablets() {
9898
}
9999

100100
public List<Long> getTabletIdsInOrder() {
101-
List<Long> tabletIds = Lists.newArrayList();
101+
List<Long> tabletIds = Lists.newArrayListWithCapacity(tablets.size());
102102
for (Tablet tablet : tablets) {
103103
tabletIds.add(tablet.getId());
104104
}

‎fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -236,21 +236,29 @@ default Map<String, Constraint> getConstraintsMap() {
236236

237237
default Set<PrimaryKeyConstraint> getPrimaryKeyConstraints() {
238238
try {
239-
return getConstraintsMapUnsafe().values().stream()
240-
.filter(PrimaryKeyConstraint.class::isInstance)
241-
.map(PrimaryKeyConstraint.class::cast)
242-
.collect(ImmutableSet.toImmutableSet());
239+
ImmutableSet.Builder<PrimaryKeyConstraint> constraintBuilder = ImmutableSet.builder();
240+
for (Constraint constraint : getConstraintsMapUnsafe().values()) {
241+
if (!(constraint instanceof PrimaryKeyConstraint)) {
242+
continue;
243+
}
244+
constraintBuilder.add((PrimaryKeyConstraint) constraint);
245+
}
246+
return constraintBuilder.build();
243247
} catch (Exception ignored) {
244248
return ImmutableSet.of();
245249
}
246250
}
247251

248252
default Set<UniqueConstraint> getUniqueConstraints() {
249253
try {
250-
return getConstraintsMapUnsafe().values().stream()
251-
.filter(UniqueConstraint.class::isInstance)
252-
.map(UniqueConstraint.class::cast)
253-
.collect(ImmutableSet.toImmutableSet());
254+
ImmutableSet.Builder<UniqueConstraint> constraintBuilder = ImmutableSet.builder();
255+
for (Constraint constraint : getConstraintsMapUnsafe().values()) {
256+
if (!(constraint instanceof UniqueConstraint)) {
257+
continue;
258+
}
259+
constraintBuilder.add((UniqueConstraint) constraint);
260+
}
261+
return constraintBuilder.build();
254262
} catch (Exception ignored) {
255263
return ImmutableSet.of();
256264
}
@@ -493,9 +501,9 @@ default List<String> getFullQualifiers() {
493501
}
494502

495503
default String getNameWithFullQualifiers() {
496-
return String.format("%s.%s.%s", getDatabase().getCatalog().getName(),
497-
ClusterNamespace.getNameFromFullName(getDatabase().getFullName()),
498-
getName());
504+
return getDatabase().getCatalog().getName()
505+
+ "." + ClusterNamespace.getNameFromFullName(getDatabase().getFullName())
506+
+ "." + getName();
499507
}
500508

501509
default boolean isManagedTable() {

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -303,11 +303,12 @@ protected Multimap<Long, Long> getNormalReplicaBackendPathMapCloud(String beEndp
303303
// for query
304304
public List<Replica> getQueryableReplicas(long visibleVersion, Map<Long, Set<Long>> backendAlivePathHashs,
305305
boolean allowMissingVersion) {
306-
List<Replica> allQueryableReplica = Lists.newArrayListWithCapacity(replicas.size());
307-
List<Replica> auxiliaryReplica = Lists.newArrayListWithCapacity(replicas.size());
308-
List<Replica> deadPathReplica = Lists.newArrayList();
309-
List<Replica> mayMissingVersionReplica = Lists.newArrayList();
310-
List<Replica> notCatchupReplica = Lists.newArrayList();
306+
int replicaNum = replicas.size();
307+
List<Replica> allQueryableReplica = Lists.newArrayListWithCapacity(replicaNum);
308+
List<Replica> auxiliaryReplica = Lists.newArrayListWithCapacity(replicaNum);
309+
List<Replica> deadPathReplica = Lists.newArrayListWithCapacity(replicaNum);
310+
List<Replica> mayMissingVersionReplica = Lists.newArrayListWithCapacity(replicaNum);
311+
List<Replica> notCatchupReplica = Lists.newArrayListWithCapacity(replicaNum);
311312

312313
for (Replica replica : replicas) {
313314
if (replica.isBad()) {
@@ -354,17 +355,25 @@ public List<Replica> getQueryableReplicas(long visibleVersion, Map<Long, Set<Lon
354355
}
355356

356357
if (Config.skip_compaction_slower_replica && allQueryableReplica.size() > 1) {
357-
long minVersionCount = allQueryableReplica.stream().mapToLong(Replica::getVisibleVersionCount)
358-
.filter(count -> count != -1).min().orElse(Long.MAX_VALUE);
358+
long minVersionCount = Long.MAX_VALUE;
359+
for (Replica replica : allQueryableReplica) {
360+
long visibleVersionCount = replica.getVisibleVersionCount();
361+
if (visibleVersionCount != 0 && visibleVersionCount < minVersionCount) {
362+
minVersionCount = visibleVersionCount;
363+
}
364+
}
359365
long maxVersionCount = Config.min_version_count_indicate_replica_compaction_too_slow;
360366
if (minVersionCount != Long.MAX_VALUE) {
361367
maxVersionCount = Math.max(maxVersionCount, minVersionCount * QUERYABLE_TIMES_OF_MIN_VERSION_COUNT);
362368
}
363369

364-
final long finalMaxVersionCount = maxVersionCount;
365-
return allQueryableReplica.stream()
366-
.filter(replica -> replica.getVisibleVersionCount() < finalMaxVersionCount)
367-
.collect(Collectors.toList());
370+
List<Replica> lowerVersionReplicas = Lists.newArrayListWithCapacity(allQueryableReplica.size());
371+
for (Replica replica : allQueryableReplica) {
372+
if (replica.getVisibleVersionCount() < maxVersionCount) {
373+
lowerVersionReplicas.add(replica);
374+
}
375+
}
376+
return lowerVersionReplicas;
368377
}
369378
return allQueryableReplica;
370379
}

‎fe/fe-core/src/main/java/org/apache/doris/cluster/ClusterNamespace.java

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,15 @@ private static boolean checkName(String str) {
4343
if (Strings.isNullOrEmpty(str)) {
4444
return false;
4545
}
46-
final String[] ele = str.split(CLUSTER_DELIMITER);
47-
return (ele.length > 1) ? true : false;
46+
47+
char delimiter = CLUSTER_DELIMITER.charAt(0);
48+
int delimiterNum = 0;
49+
for (int i = 0; i < str.length(); i++) {
50+
if (str.charAt(i) == delimiter) {
51+
delimiterNum++;
52+
}
53+
}
54+
return delimiterNum >= 1;
4855
}
4956

5057
private static String linkString(String cluster, String name) {

‎fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.apache.doris.thrift.TUniqueId;
2424

2525
import com.google.common.base.Strings;
26+
import org.apache.commons.lang3.StringUtils;
2627

2728
import java.io.PrintWriter;
2829
import java.io.StringWriter;
@@ -206,7 +207,7 @@ public static String prettyPrintChangedSessionVar(List<List<String>> nestedList)
206207

207208
// Build the table header
208209
for (int i = 0; i < headers.length; i++) {
209-
output.append(String.format("%-" + columnWidths[i] + "s", headers[i]));
210+
output.append(format(columnWidths[i], headers[i]));
210211
if (i < headers.length - 1) {
211212
output.append(" | "); // Separator between columns
212213
}
@@ -215,7 +216,7 @@ public static String prettyPrintChangedSessionVar(List<List<String>> nestedList)
215216

216217
// Add a separator line for better readability (optional)
217218
for (int i = 0; i < headers.length; i++) {
218-
output.append(String.format("%-" + columnWidths[i] + "s", Strings.repeat("-", columnWidths[i])));
219+
output.append(format(columnWidths[i], Strings.repeat("-", columnWidths[i])));
219220
if (i < headers.length - 1) {
220221
output.append("-|-"); // Separator between columns
221222
}
@@ -227,7 +228,7 @@ public static String prettyPrintChangedSessionVar(List<List<String>> nestedList)
227228
for (int i = 0; i < row.size(); i++) {
228229
String element = row.get(i);
229230
// Pad with spaces if the element is shorter than the column width
230-
output.append(String.format("%-" + columnWidths[i] + "s", element));
231+
output.append(format(columnWidths[i], element));
231232
if (i < row.size() - 1) {
232233
output.append(" | "); // Separator between columns
233234
}
@@ -269,4 +270,8 @@ public static String prettyPrintPlanNodeRuntimeStatsItems(
269270
}
270271
return result.toString();
271272
}
273+
274+
private static String format(int width, String name) {
275+
return name + StringUtils.repeat(" ", Math.max(0, name.length() - width));
276+
}
272277
}

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
3434
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
3535
import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink;
36-
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
3736
import org.apache.doris.qe.ConnectContext;
3837
import org.apache.doris.qe.OriginStatement;
3938
import org.apache.doris.statistics.Statistics;
@@ -127,13 +126,12 @@ public static MTMVCache from(String defSql,
127126
originPlan = planner.getCascadesContext().getRewritePlan();
128127
// Eliminate result sink because sink operator is useless in query rewrite by materialized view
129128
// and the top sort can also be removed
130-
mvPlan = originPlan.accept(new DefaultPlanRewriter<Object>() {
131-
@Override
132-
public Plan visitLogicalResultSink(LogicalResultSink<? extends Plan> logicalResultSink,
133-
Object context) {
134-
return logicalResultSink.child().accept(this, context);
129+
mvPlan = originPlan.rewriteDownShortCircuit(p -> {
130+
if (p instanceof LogicalResultSink) {
131+
return p.child(0);
135132
}
136-
}, null);
133+
return p;
134+
});
137135
// Optimize by rules to remove top sort
138136
CascadesContext parentCascadesContext = CascadesContext.initContext(mvSqlStatementContext, mvPlan,
139137
PhysicalProperties.ANY);

‎fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -542,6 +542,10 @@ public boolean isEnableExprTrace() {
542542
return isEnableExprTrace;
543543
}
544544

545+
public boolean rewritePlanContainsTypes(Class<?>... types) {
546+
return getRewritePlan().containsType(types);
547+
}
548+
545549
public RuntimeFilterContextV2 getRuntimeFilterV2Context() {
546550
return runtimeFilterV2Context;
547551
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@
105105
* Planner to do query plan in Nereids.
106106
*/
107107
public class NereidsPlanner extends Planner {
108+
// private static final AtomicInteger executeCount = new AtomicInteger(0);
108109
public static final Logger LOG = LogManager.getLogger(NereidsPlanner.class);
109110

110111
protected Plan parsedPlan;
@@ -255,7 +256,6 @@ private Plan planWithoutLock(
255256
boolean showPlanProcess) {
256257
// minidump of input must be serialized first, this process ensure minidump string not null
257258
try {
258-
259259
MinidumpUtils.serializeInputsToDumpFile(plan, statementContext);
260260
} catch (IOException e) {
261261
throw new RuntimeException(e);
@@ -278,7 +278,7 @@ private Plan planWithoutLock(
278278
}
279279
}
280280

281-
optimize();
281+
optimize(showPlanProcess);
282282
// print memo before choose plan.
283283
// if chooseNthPlan failed, we could get memo to debug
284284
if (cascadesContext.getConnectContext().getSessionVariable().dumpNereidsMemo) {
@@ -403,7 +403,9 @@ protected void rewrite(boolean showPlanProcess) {
403403
if (LOG.isDebugEnabled()) {
404404
LOG.debug("Start rewrite plan");
405405
}
406-
keepOrShowPlanProcess(showPlanProcess, () -> Rewriter.getWholeTreeRewriter(cascadesContext).execute());
406+
keepOrShowPlanProcess(showPlanProcess, () -> {
407+
Rewriter.getWholeTreeRewriter(cascadesContext).execute();
408+
});
407409
NereidsTracer.logImportantTime("EndRewritePlan");
408410
if (LOG.isDebugEnabled()) {
409411
LOG.debug("End rewrite plan");
@@ -414,8 +416,8 @@ protected void rewrite(boolean showPlanProcess) {
414416
cascadesContext.getStatementContext().getPlannerHooks().forEach(hook -> hook.afterRewrite(this));
415417
}
416418

417-
// DependsRules: EnsureProjectOnTopJoin.class
418-
protected void optimize() {
419+
// DependsRules: AddProjectForJoin
420+
protected void optimize(boolean showPlanProcess) {
419421
// if we cannot get table row count, skip join reorder
420422
// except:
421423
// 1. user set leading hint
@@ -432,7 +434,9 @@ protected void optimize() {
432434
if (LOG.isDebugEnabled()) {
433435
LOG.debug("Start optimize plan");
434436
}
435-
new Optimizer(cascadesContext).execute();
437+
keepOrShowPlanProcess(showPlanProcess, () -> {
438+
new Optimizer(cascadesContext).execute();
439+
});
436440
NereidsTracer.logImportantTime("EndOptimizePlan");
437441
if (LOG.isDebugEnabled()) {
438442
LOG.debug("End optimize plan");
@@ -583,13 +587,22 @@ protected void distribute(PhysicalPlan physicalPlan, ExplainLevel explainLevel)
583587
}
584588

585589
splitFragments(physicalPlan);
586-
doDistribute(canUseNereidsDistributePlanner);
590+
doDistribute(canUseNereidsDistributePlanner, explainLevel);
587591
}
588592

589-
protected void doDistribute(boolean canUseNereidsDistributePlanner) {
593+
protected void doDistribute(boolean canUseNereidsDistributePlanner, ExplainLevel explainLevel) {
590594
if (!canUseNereidsDistributePlanner) {
591595
return;
592596
}
597+
switch (explainLevel) {
598+
case NONE:
599+
case ALL_PLAN:
600+
case DISTRIBUTED_PLAN:
601+
break;
602+
default: {
603+
return;
604+
}
605+
}
593606

594607
boolean notNeedBackend = false;
595608
// if the query can compute without backend, we can skip check cluster privileges

‎fe/fe-core/src/main/java/org/apache/doris/nereids/PlanProcess.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,4 +28,9 @@ public PlanProcess(String ruleName, String beforeShape, String afterShape) {
2828
this.beforeShape = beforeShape;
2929
this.afterShape = afterShape;
3030
}
31+
32+
@Override
33+
public String toString() {
34+
return "PlanProcess{ruleName='" + ruleName + '\'' + '}';
35+
}
3136
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -452,6 +452,10 @@ public ExprId getNextExprId() {
452452
return exprIdGenerator.getNextId();
453453
}
454454

455+
public IdGenerator<ExprId> getExprIdGenerator() {
456+
return exprIdGenerator;
457+
}
458+
455459
public CTEId getNextCTEId() {
456460
return cteIdGenerator.getNextId();
457461
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,28 @@ public String toString() {
187187
return Utils.toSqlString("UnboundRelation", args.toArray());
188188
}
189189

190+
@Override
191+
public boolean equals(Object o) {
192+
if (o == null || getClass() != o.getClass()) {
193+
return false;
194+
}
195+
if (!super.equals(o)) {
196+
return false;
197+
}
198+
UnboundRelation that = (UnboundRelation) o;
199+
return isTempPart == that.isTempPart && Objects.equals(nameParts, that.nameParts)
200+
&& Objects.equals(partNames, that.partNames) && Objects.equals(tabletIds,
201+
that.tabletIds) && Objects.equals(hints, that.hints) && Objects.equals(tableSample,
202+
that.tableSample) && Objects.equals(indexName, that.indexName) && Objects.equals(
203+
scanParams, that.scanParams) && Objects.equals(indexInSqlString, that.indexInSqlString)
204+
&& Objects.equals(tableSnapshot, that.tableSnapshot);
205+
}
206+
207+
@Override
208+
public int hashCode() {
209+
return Objects.hash(super.hashCode());
210+
}
211+
190212
@Override
191213
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
192214
return visitor.visitUnboundRelation(this, context);

0 commit comments

Comments
 (0)