Skip to content

Commit 0d589fd

Browse files
924060929924060929
committed
opt small query
1 parent 15c16ad commit 0d589fd

File tree

323 files changed

+5640
-2599
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

323 files changed

+5640
-2599
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -377,7 +377,6 @@ public void setMutableState(String key, Object value) {
377377
@Override
378378
public Expression withChildren(List<Expression> children) {
379379
throw new AnalysisException("could not call withChildren on UdfSignatureSearcher");
380-
381380
}
382381
}
383382
}

‎fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ public List<Tablet> getTablets() {
9898
}
9999

100100
public List<Long> getTabletIdsInOrder() {
101-
List<Long> tabletIds = Lists.newArrayList();
101+
List<Long> tabletIds = Lists.newArrayListWithCapacity(tablets.size());
102102
for (Tablet tablet : tablets) {
103103
tabletIds.add(tablet.getId());
104104
}

‎fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -236,21 +236,29 @@ default Map<String, Constraint> getConstraintsMap() {
236236

237237
default Set<PrimaryKeyConstraint> getPrimaryKeyConstraints() {
238238
try {
239-
return getConstraintsMapUnsafe().values().stream()
240-
.filter(PrimaryKeyConstraint.class::isInstance)
241-
.map(PrimaryKeyConstraint.class::cast)
242-
.collect(ImmutableSet.toImmutableSet());
239+
ImmutableSet.Builder<PrimaryKeyConstraint> constraintBuilder = ImmutableSet.builder();
240+
for (Constraint constraint : getConstraintsMapUnsafe().values()) {
241+
if (!(constraint instanceof PrimaryKeyConstraint)) {
242+
continue;
243+
}
244+
constraintBuilder.add((PrimaryKeyConstraint) constraint);
245+
}
246+
return constraintBuilder.build();
243247
} catch (Exception ignored) {
244248
return ImmutableSet.of();
245249
}
246250
}
247251

248252
default Set<UniqueConstraint> getUniqueConstraints() {
249253
try {
250-
return getConstraintsMapUnsafe().values().stream()
251-
.filter(UniqueConstraint.class::isInstance)
252-
.map(UniqueConstraint.class::cast)
253-
.collect(ImmutableSet.toImmutableSet());
254+
ImmutableSet.Builder<UniqueConstraint> constraintBuilder = ImmutableSet.builder();
255+
for (Constraint constraint : getConstraintsMapUnsafe().values()) {
256+
if (!(constraint instanceof UniqueConstraint)) {
257+
continue;
258+
}
259+
constraintBuilder.add((UniqueConstraint) constraint);
260+
}
261+
return constraintBuilder.build();
254262
} catch (Exception ignored) {
255263
return ImmutableSet.of();
256264
}
@@ -493,9 +501,9 @@ default List<String> getFullQualifiers() {
493501
}
494502

495503
default String getNameWithFullQualifiers() {
496-
return String.format("%s.%s.%s", getDatabase().getCatalog().getName(),
497-
ClusterNamespace.getNameFromFullName(getDatabase().getFullName()),
498-
getName());
504+
return getDatabase().getCatalog().getName()
505+
+ "." + ClusterNamespace.getNameFromFullName(getDatabase().getFullName())
506+
+ "." + getName();
499507
}
500508

501509
default boolean isManagedTable() {

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -303,11 +303,12 @@ protected Multimap<Long, Long> getNormalReplicaBackendPathMapCloud(String beEndp
303303
// for query
304304
public List<Replica> getQueryableReplicas(long visibleVersion, Map<Long, Set<Long>> backendAlivePathHashs,
305305
boolean allowMissingVersion) {
306-
List<Replica> allQueryableReplica = Lists.newArrayListWithCapacity(replicas.size());
307-
List<Replica> auxiliaryReplica = Lists.newArrayListWithCapacity(replicas.size());
308-
List<Replica> deadPathReplica = Lists.newArrayList();
309-
List<Replica> mayMissingVersionReplica = Lists.newArrayList();
310-
List<Replica> notCatchupReplica = Lists.newArrayList();
306+
int replicaNum = replicas.size();
307+
List<Replica> allQueryableReplica = Lists.newArrayListWithCapacity(replicaNum);
308+
List<Replica> auxiliaryReplica = Lists.newArrayListWithCapacity(replicaNum);
309+
List<Replica> deadPathReplica = Lists.newArrayListWithCapacity(replicaNum);
310+
List<Replica> mayMissingVersionReplica = Lists.newArrayListWithCapacity(replicaNum);
311+
List<Replica> notCatchupReplica = Lists.newArrayListWithCapacity(replicaNum);
311312

312313
for (Replica replica : replicas) {
313314
if (replica.isBad()) {
@@ -354,17 +355,25 @@ public List<Replica> getQueryableReplicas(long visibleVersion, Map<Long, Set<Lon
354355
}
355356

356357
if (Config.skip_compaction_slower_replica && allQueryableReplica.size() > 1) {
357-
long minVersionCount = allQueryableReplica.stream().mapToLong(Replica::getVisibleVersionCount)
358-
.filter(count -> count != -1).min().orElse(Long.MAX_VALUE);
358+
long minVersionCount = Long.MAX_VALUE;
359+
for (Replica replica : allQueryableReplica) {
360+
long visibleVersionCount = replica.getVisibleVersionCount();
361+
if (visibleVersionCount != 0 && visibleVersionCount < minVersionCount) {
362+
minVersionCount = visibleVersionCount;
363+
}
364+
}
359365
long maxVersionCount = Config.min_version_count_indicate_replica_compaction_too_slow;
360366
if (minVersionCount != Long.MAX_VALUE) {
361367
maxVersionCount = Math.max(maxVersionCount, minVersionCount * QUERYABLE_TIMES_OF_MIN_VERSION_COUNT);
362368
}
363369

364-
final long finalMaxVersionCount = maxVersionCount;
365-
return allQueryableReplica.stream()
366-
.filter(replica -> replica.getVisibleVersionCount() < finalMaxVersionCount)
367-
.collect(Collectors.toList());
370+
List<Replica> lowerVersionReplicas = Lists.newArrayListWithCapacity(allQueryableReplica.size());
371+
for (Replica replica : allQueryableReplica) {
372+
if (replica.getVisibleVersionCount() < maxVersionCount) {
373+
lowerVersionReplicas.add(replica);
374+
}
375+
}
376+
return lowerVersionReplicas;
368377
}
369378
return allQueryableReplica;
370379
}

‎fe/fe-core/src/main/java/org/apache/doris/cluster/ClusterNamespace.java

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,15 @@ private static boolean checkName(String str) {
4343
if (Strings.isNullOrEmpty(str)) {
4444
return false;
4545
}
46-
final String[] ele = str.split(CLUSTER_DELIMITER);
47-
return (ele.length > 1) ? true : false;
46+
47+
char delimiter = CLUSTER_DELIMITER.charAt(0);
48+
int delimiterNum = 0;
49+
for (int i = 0; i < str.length(); i++) {
50+
if (str.charAt(i) == delimiter) {
51+
delimiterNum++;
52+
}
53+
}
54+
return delimiterNum >= 1;
4855
}
4956

5057
private static String linkString(String cluster, String name) {

‎fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.apache.doris.thrift.TUniqueId;
2424

2525
import com.google.common.base.Strings;
26+
import org.apache.commons.lang3.StringUtils;
2627

2728
import java.io.PrintWriter;
2829
import java.io.StringWriter;
@@ -206,7 +207,7 @@ public static String prettyPrintChangedSessionVar(List<List<String>> nestedList)
206207

207208
// Build the table header
208209
for (int i = 0; i < headers.length; i++) {
209-
output.append(String.format("%-" + columnWidths[i] + "s", headers[i]));
210+
output.append(format(columnWidths[i], headers[i]));
210211
if (i < headers.length - 1) {
211212
output.append(" | "); // Separator between columns
212213
}
@@ -215,7 +216,7 @@ public static String prettyPrintChangedSessionVar(List<List<String>> nestedList)
215216

216217
// Add a separator line for better readability (optional)
217218
for (int i = 0; i < headers.length; i++) {
218-
output.append(String.format("%-" + columnWidths[i] + "s", Strings.repeat("-", columnWidths[i])));
219+
output.append(format(columnWidths[i], Strings.repeat("-", columnWidths[i])));
219220
if (i < headers.length - 1) {
220221
output.append("-|-"); // Separator between columns
221222
}
@@ -227,7 +228,7 @@ public static String prettyPrintChangedSessionVar(List<List<String>> nestedList)
227228
for (int i = 0; i < row.size(); i++) {
228229
String element = row.get(i);
229230
// Pad with spaces if the element is shorter than the column width
230-
output.append(String.format("%-" + columnWidths[i] + "s", element));
231+
output.append(format(columnWidths[i], element));
231232
if (i < row.size() - 1) {
232233
output.append(" | "); // Separator between columns
233234
}
@@ -269,4 +270,8 @@ public static String prettyPrintPlanNodeRuntimeStatsItems(
269270
}
270271
return result.toString();
271272
}
273+
274+
private static String format(int width, String name) {
275+
return name + StringUtils.repeat(" ", Math.max(0, name.length() - width));
276+
}
272277
}

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
3434
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
3535
import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink;
36-
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
3736
import org.apache.doris.qe.ConnectContext;
3837
import org.apache.doris.qe.OriginStatement;
3938
import org.apache.doris.statistics.Statistics;
@@ -127,13 +126,12 @@ public static MTMVCache from(String defSql,
127126
originPlan = planner.getCascadesContext().getRewritePlan();
128127
// Eliminate result sink because sink operator is useless in query rewrite by materialized view
129128
// and the top sort can also be removed
130-
mvPlan = originPlan.accept(new DefaultPlanRewriter<Object>() {
131-
@Override
132-
public Plan visitLogicalResultSink(LogicalResultSink<? extends Plan> logicalResultSink,
133-
Object context) {
134-
return logicalResultSink.child().accept(this, context);
129+
mvPlan = originPlan.rewriteDownShortCircuit(p -> {
130+
if (p instanceof LogicalResultSink) {
131+
return p.child(0);
135132
}
136-
}, null);
133+
return p;
134+
});
137135
// Optimize by rules to remove top sort
138136
CascadesContext parentCascadesContext = CascadesContext.initContext(mvSqlStatementContext, mvPlan,
139137
PhysicalProperties.ANY);

‎fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -534,4 +534,8 @@ public int getDistinctAggLevel() {
534534
public boolean isEnableExprTrace() {
535535
return isEnableExprTrace;
536536
}
537+
538+
public boolean rewritePlanContainsTypes(Class<?>... types) {
539+
return getRewritePlan().containsType(types);
540+
}
537541
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@
105105
* Planner to do query plan in Nereids.
106106
*/
107107
public class NereidsPlanner extends Planner {
108+
// private static final AtomicInteger executeCount = new AtomicInteger(0);
108109
public static final Logger LOG = LogManager.getLogger(NereidsPlanner.class);
109110

110111
protected Plan parsedPlan;
@@ -255,7 +256,6 @@ private Plan planWithoutLock(
255256
boolean showPlanProcess) {
256257
// minidump of input must be serialized first, this process ensure minidump string not null
257258
try {
258-
259259
MinidumpUtils.serializeInputsToDumpFile(plan, statementContext);
260260
} catch (IOException e) {
261261
throw new RuntimeException(e);
@@ -278,7 +278,7 @@ private Plan planWithoutLock(
278278
}
279279
}
280280

281-
optimize();
281+
optimize(showPlanProcess);
282282
// print memo before choose plan.
283283
// if chooseNthPlan failed, we could get memo to debug
284284
if (cascadesContext.getConnectContext().getSessionVariable().dumpNereidsMemo) {
@@ -403,7 +403,9 @@ protected void rewrite(boolean showPlanProcess) {
403403
if (LOG.isDebugEnabled()) {
404404
LOG.debug("Start rewrite plan");
405405
}
406-
keepOrShowPlanProcess(showPlanProcess, () -> Rewriter.getWholeTreeRewriter(cascadesContext).execute());
406+
keepOrShowPlanProcess(showPlanProcess, () -> {
407+
Rewriter.getWholeTreeRewriter(cascadesContext).execute();
408+
});
407409
NereidsTracer.logImportantTime("EndRewritePlan");
408410
if (LOG.isDebugEnabled()) {
409411
LOG.debug("End rewrite plan");
@@ -414,8 +416,8 @@ protected void rewrite(boolean showPlanProcess) {
414416
cascadesContext.getStatementContext().getPlannerHooks().forEach(hook -> hook.afterRewrite(this));
415417
}
416418

417-
// DependsRules: EnsureProjectOnTopJoin.class
418-
protected void optimize() {
419+
// DependsRules: AddProjectForJoin
420+
protected void optimize(boolean showPlanProcess) {
419421
// if we cannot get table row count, skip join reorder
420422
// except:
421423
// 1. user set leading hint
@@ -432,7 +434,9 @@ protected void optimize() {
432434
if (LOG.isDebugEnabled()) {
433435
LOG.debug("Start optimize plan");
434436
}
435-
new Optimizer(cascadesContext).execute();
437+
keepOrShowPlanProcess(showPlanProcess, () -> {
438+
new Optimizer(cascadesContext).execute();
439+
});
436440
NereidsTracer.logImportantTime("EndOptimizePlan");
437441
if (LOG.isDebugEnabled()) {
438442
LOG.debug("End optimize plan");
@@ -583,13 +587,22 @@ protected void distribute(PhysicalPlan physicalPlan, ExplainLevel explainLevel)
583587
}
584588

585589
splitFragments(physicalPlan);
586-
doDistribute(canUseNereidsDistributePlanner);
590+
doDistribute(canUseNereidsDistributePlanner, explainLevel);
587591
}
588592

589-
protected void doDistribute(boolean canUseNereidsDistributePlanner) {
593+
protected void doDistribute(boolean canUseNereidsDistributePlanner, ExplainLevel explainLevel) {
590594
if (!canUseNereidsDistributePlanner) {
591595
return;
592596
}
597+
switch (explainLevel) {
598+
case NONE:
599+
case ALL_PLAN:
600+
case DISTRIBUTED_PLAN:
601+
break;
602+
default: {
603+
return;
604+
}
605+
}
593606

594607
boolean notNeedBackend = false;
595608
// if the query can compute without backend, we can skip check cluster privileges

‎fe/fe-core/src/main/java/org/apache/doris/nereids/PlanProcess.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,4 +28,9 @@ public PlanProcess(String ruleName, String beforeShape, String afterShape) {
2828
this.beforeShape = beforeShape;
2929
this.afterShape = afterShape;
3030
}
31+
32+
@Override
33+
public String toString() {
34+
return "PlanProcess{ruleName='" + ruleName + '\'' + '}';
35+
}
3136
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -450,6 +450,10 @@ public ExprId getNextExprId() {
450450
return exprIdGenerator.getNextId();
451451
}
452452

453+
public IdGenerator<ExprId> getExprIdGenerator() {
454+
return exprIdGenerator;
455+
}
456+
453457
public CTEId getNextCTEId() {
454458
return cteIdGenerator.getNextId();
455459
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,28 @@ public String toString() {
195195
return Utils.toSqlString("UnboundRelation", args.toArray());
196196
}
197197

198+
@Override
199+
public boolean equals(Object o) {
200+
if (o == null || getClass() != o.getClass()) {
201+
return false;
202+
}
203+
if (!super.equals(o)) {
204+
return false;
205+
}
206+
UnboundRelation that = (UnboundRelation) o;
207+
return isTempPart == that.isTempPart && Objects.equals(nameParts, that.nameParts)
208+
&& Objects.equals(partNames, that.partNames) && Objects.equals(tabletIds,
209+
that.tabletIds) && Objects.equals(hints, that.hints) && Objects.equals(tableSample,
210+
that.tableSample) && Objects.equals(indexName, that.indexName) && Objects.equals(
211+
scanParams, that.scanParams) && Objects.equals(indexInSqlString, that.indexInSqlString)
212+
&& Objects.equals(tableSnapshot, that.tableSnapshot);
213+
}
214+
215+
@Override
216+
public int hashCode() {
217+
return Objects.hash(super.hashCode());
218+
}
219+
198220
@Override
199221
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
200222
return visitor.visitUnboundRelation(this, context);

0 commit comments

Comments
 (0)