[pinpoint-apm#12434] Optimize scanner caching

Author: emeroad

commons-timeseries/src/main/java/com/navercorp/pinpoint/common/timeseries/window/DefaultTimeSlot.java

@@ -24,7 +24,7 @@
  */
 public class DefaultTimeSlot implements TimeSlot {
 
-    private static final long ONE_MIN_RESOLUTION =  60000; // 1min
+    public static final long ONE_MIN_RESOLUTION =  60000; // 1min
 
     private final long resolution;
 

commons-timeseries/src/main/java/com/navercorp/pinpoint/common/timeseries/window/TimeWindow.java

@@ -47,7 +47,7 @@ public TimeWindow(Range range, TimeWindowSampler sampler) {
     TimeWindow(long rangeFrom, long rangeTo, TimeWindowSampler sampler) {
         Objects.requireNonNull(sampler, "sampler");
         this.windowSlotSize = sampler.getWindowSize(duration(rangeTo, rangeFrom));
-        this.from = refineTimestamp(rangeFrom);
+        this.from = this.refineTimestamp(rangeFrom);
         this.to = this.refineTimestamp(rangeTo);
         this.windowRangeCount = computeWindowRangeCount(duration(to, from), windowSlotSize);
     }
@@ -111,4 +111,13 @@ public int getWindowIndex(long time) {
         return (int) index;
     }
 
+    @Override
+    public String toString() {
+        return "TimeWindow{" +
+                "windowSlotSize=" + windowSlotSize +
+                ", from=" + from +
+                ", to=" + to +
+                ", windowRangeCount=" + windowRangeCount +
+                '}';
+    }
 }

web/src/main/java/com/navercorp/pinpoint/web/applicationmap/dao/hbase/MapScanFactory.java

@@ -1,7 +1,9 @@
 package com.navercorp.pinpoint.web.applicationmap.dao.hbase;
 
+import com.google.common.primitives.Ints;
 import com.navercorp.pinpoint.common.server.util.ApplicationMapStatisticsUtils;
 import com.navercorp.pinpoint.common.timeseries.time.Range;
+import com.navercorp.pinpoint.common.timeseries.window.DefaultTimeSlot;
 import com.navercorp.pinpoint.web.vo.Application;
 import com.navercorp.pinpoint.web.vo.RangeFactory;
 import org.apache.hadoop.hbase.client.Scan;
@@ -13,40 +15,50 @@
 public class MapScanFactory {
     private final Logger logger = LogManager.getLogger(this.getClass());
 
-    public static final int SCAN_CACHE_SIZE = 40;
+    public static final int SCAN_CACHE_SIZE_MIN = 8;
+    public static final int SCAN_CACHE_SIZE_MAX = 1024;
 
-    private final RangeFactory rangeFactory;
+    private long slotSize = DefaultTimeSlot.ONE_MIN_RESOLUTION;
 
-    private int scanCacheSize;
+    private final RangeFactory rangeFactory;
 
     public MapScanFactory(RangeFactory rangeFactory) {
         this.rangeFactory = Objects.requireNonNull(rangeFactory, "rangeFactory");
-        this.scanCacheSize = SCAN_CACHE_SIZE;
     }
 
-    public void setScanCacheSize(int scanCacheSize) {
-        this.scanCacheSize = scanCacheSize;
+    public void setSlotSize(long slotSize) {
+        this.slotSize = slotSize;
     }
 
     public Scan createScan(String id, Application application, Range range, byte[] family) {
         range = rangeFactory.createStatisticsRange(range);
         if (logger.isDebugEnabled()) {
             logger.debug("scan time:{} ", range.prettyToString());
         }
-
         // start key is replaced by end key because timestamp has been reversed
         byte[] startKey = ApplicationMapStatisticsUtils
                 .makeRowKey(application.getName(), (short) application.getServiceTypeCode(), range.getTo());
         byte[] endKey = ApplicationMapStatisticsUtils
                 .makeRowKey(application.getName(), (short) application.getServiceTypeCode(), range.getFrom());
 
         final Scan scan = new Scan();
-        scan.setCaching(this.scanCacheSize);
+
+        final int scannerCaching = computeScannerCaching(range);
+        scan.setCaching(scannerCaching);
         scan.withStartRow(startKey);
         scan.withStopRow(endKey);
         scan.addFamily(family);
         scan.setId(id);
 
         return scan;
     }
+
+    private int computeScannerCaching(Range range) {
+        int windowCount = (int)(range.durationMillis() / slotSize) + 1;
+        int scannerCaching = Ints.constrainToRange(windowCount, SCAN_CACHE_SIZE_MIN, SCAN_CACHE_SIZE_MAX);
+        if (logger.isDebugEnabled()) {
+            logger.debug("ScannerCaching:{} windowCount:{}", scannerCaching, windowCount);
+        }
+        return scannerCaching;
+    }
 }

web/src/main/java/com/navercorp/pinpoint/web/applicationmap/nodes/ServerBuilder.java

@@ -20,10 +20,12 @@
 import com.navercorp.pinpoint.web.applicationmap.rawdata.AgentHistogram;
 import com.navercorp.pinpoint.web.applicationmap.rawdata.AgentHistogramList;
 import com.navercorp.pinpoint.web.vo.agent.AgentAndStatus;
+import com.navercorp.pinpoint.web.vo.agent.AgentInfo;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 
 /**
@@ -97,7 +99,13 @@ public ServerGroupList buildPhysicalServer(final Set<AgentAndStatus> agentSet) {
     public ServerGroupList build() {
         if (!agentSet.isEmpty()) {
             // if agent name exists (physical server exists)
-            this.logger.debug("buildPhysicalServer:{}", agentSet);
+            if (logger.isDebugEnabled()) {
+                List<String> agents = agentSet.stream()
+                        .map(AgentAndStatus::getAgentInfo)
+                        .map(AgentInfo::getAgentId)
+                        .toList();
+                this.logger.debug("buildPhysicalServer: {}", agents);
+            }
             return buildPhysicalServer(agentSet);
         } else {
             // otherwise, create logical name

web/src/main/java/com/navercorp/pinpoint/web/applicationmap/service/MapServiceImpl.java

@@ -130,7 +130,7 @@ public ApplicationMap selectApplicationMap(MapServiceOption option) {
         }
         watch.stop();
         if (logger.isInfoEnabled()) {
-            logger.info("ApplicationMap BuildTime: {}", watch.prettyPrint());
+            logger.info("ApplicationMap BuildTime: {} {}", option.getSourceApplication(), watch.prettyPrint());
         }
         if (serverMapDataFilter != null) {
             map = serverMapDataFilter.dataFiltering(map);