Skip to content

Commit cd76c73

Browse files
surbhigarg92cloud-java-bot
surbhigarg92
and
cloud-java-bot
authored
feat: [Internal] open telemetry built in metrics for GRPC (#3709)
* feat: grpc metrics * test * grpc metrics env check * skip test and clirr * add grpc metrics in allowed metrics * test: logs * chore: generate libraries at Wed Apr 2 17:15:18 UTC 2025 * chore: generate libraries at Wed Apr 2 17:18:00 UTC 2025 * remove logs * review * clirr * review comments * enable grpc metrics only env is set to false --------- Co-authored-by: cloud-java-bot <[email protected]>
1 parent 1afd815 commit cd76c73

11 files changed

+244
-168
lines changed

‎google-cloud-spanner/clirr-ignored-differences.xml

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -751,6 +751,13 @@
751751
<method>boolean isEnableBuiltInMetrics()</method>
752752
</difference>
753753

754+
<!-- Added Built In GRPC Metrics option -->
755+
<difference>
756+
<differenceType>7012</differenceType>
757+
<className>com/google/cloud/spanner/SpannerOptions$SpannerEnvironment</className>
758+
<method>boolean isEnableGRPCBuiltInMetrics()</method>
759+
</difference>
760+
754761
<!-- Added Monitoring host option -->
755762
<difference>
756763
<differenceType>7012</differenceType>
@@ -807,7 +814,7 @@
807814
<className>com/google/cloud/spanner/connection/Connection</className>
808815
<method>boolean isKeepTransactionAlive()</method>
809816
</difference>
810-
817+
811818
<!-- Automatic DML batching -->
812819
<difference>
813820
<differenceType>7012</differenceType>
@@ -839,7 +846,7 @@
839846
<className>com/google/cloud/spanner/connection/Connection</className>
840847
<method>boolean isAutoBatchDmlUpdateCountVerification()</method>
841848
</difference>
842-
849+
843850
<!-- Retry DML as Partitioned DML -->
844851
<difference>
845852
<differenceType>7012</differenceType>
@@ -863,7 +870,7 @@
863870
<className>com/google/cloud/spanner/connection/Connection</className>
864871
<method>java.lang.Object runTransaction(com.google.cloud.spanner.connection.Connection$TransactionCallable)</method>
865872
</difference>
866-
873+
867874
<!-- Added experimental host option -->
868875
<difference>
869876
<differenceType>7012</differenceType>

‎google-cloud-spanner/pom.xml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,10 @@
191191
<groupId>io.grpc</groupId>
192192
<artifactId>grpc-stub</artifactId>
193193
</dependency>
194+
<dependency>
195+
<groupId>io.grpc</groupId>
196+
<artifactId>grpc-opentelemetry</artifactId>
197+
</dependency>
194198
<dependency>
195199
<groupId>com.google.api</groupId>
196200
<artifactId>api-common</artifactId>

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java

Lines changed: 45 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import io.opentelemetry.sdk.metrics.InstrumentSelector;
2727
import io.opentelemetry.sdk.metrics.InstrumentType;
2828
import io.opentelemetry.sdk.metrics.View;
29+
import java.util.Collection;
2930
import java.util.Map;
3031
import java.util.Set;
3132
import java.util.stream.Collectors;
@@ -36,6 +37,7 @@ public class BuiltInMetricsConstant {
3637
public static final String METER_NAME = "spanner.googleapis.com/internal/client";
3738
public static final String GAX_METER_NAME = OpenTelemetryMetricsRecorder.GAX_METER_NAME;
3839
static final String SPANNER_METER_NAME = "spanner-java";
40+
static final String GRPC_METER_NAME = "grpc-java";
3941
static final String GFE_LATENCIES_NAME = "gfe_latencies";
4042
static final String OPERATION_LATENCIES_NAME = "operation_latencies";
4143
static final String ATTEMPT_LATENCIES_NAME = "attempt_latencies";
@@ -55,6 +57,14 @@ public class BuiltInMetricsConstant {
5557
.map(m -> METER_NAME + '/' + m)
5658
.collect(Collectors.toSet());
5759

60+
static final Collection<String> GRPC_METRICS_TO_ENABLE =
61+
ImmutableList.of(
62+
"grpc.lb.rls.default_target_picks",
63+
"grpc.lb.rls.target_picks",
64+
"grpc.xds_client.server_failure",
65+
"grpc.xds_client.resource_updates_invalid",
66+
"grpc.xds_client.resource_updates_valid");
67+
5868
public static final String SPANNER_RESOURCE_TYPE = "spanner_instance_client";
5969

6070
public static final AttributeKey<String> PROJECT_ID_KEY = AttributeKey.stringKey("project_id");
@@ -66,12 +76,7 @@ public class BuiltInMetricsConstant {
6676

6777
// These metric labels will be promoted to the spanner monitored resource fields
6878
public static final Set<AttributeKey<String>> SPANNER_PROMOTED_RESOURCE_LABELS =
69-
ImmutableSet.of(
70-
PROJECT_ID_KEY,
71-
INSTANCE_ID_KEY,
72-
INSTANCE_CONFIG_ID_KEY,
73-
LOCATION_ID_KEY,
74-
CLIENT_HASH_KEY);
79+
ImmutableSet.of(INSTANCE_ID_KEY);
7580

7681
public static final AttributeKey<String> DATABASE_KEY = AttributeKey.stringKey("database");
7782
public static final AttributeKey<String> CLIENT_UID_KEY = AttributeKey.stringKey("client_uid");
@@ -102,6 +107,9 @@ public class BuiltInMetricsConstant {
102107
DIRECT_PATH_ENABLED_KEY,
103108
DIRECT_PATH_USED_KEY);
104109

110+
static final Set<String> GRPC_LB_RLS_ATTRIBUTES =
111+
ImmutableSet.of("grpc.lb.rls.data_plane_target", "grpc.lb.pick_result");
112+
105113
static Aggregation AGGREGATION_WITH_MILLIS_HISTOGRAM =
106114
Aggregation.explicitBucketHistogram(
107115
ImmutableList.of(
@@ -111,6 +119,14 @@ public class BuiltInMetricsConstant {
111119
10000.0, 20000.0, 50000.0, 100000.0, 200000.0, 400000.0, 800000.0, 1600000.0,
112120
3200000.0));
113121

122+
static final Collection<String> GRPC_METRICS_ENABLED_BY_DEFAULT =
123+
ImmutableList.of(
124+
"grpc.client.attempt.sent_total_compressed_message_size",
125+
"grpc.client.attempt.rcvd_total_compressed_message_size",
126+
"grpc.client.attempt.started",
127+
"grpc.client.attempt.duration",
128+
"grpc.client.call.duration");
129+
114130
static Map<InstrumentSelector, View> getAllViews() {
115131
ImmutableMap.Builder<InstrumentSelector, View> views = ImmutableMap.builder();
116132
defineView(
@@ -153,6 +169,7 @@ static Map<InstrumentSelector, View> getAllViews() {
153169
Aggregation.sum(),
154170
InstrumentType.COUNTER,
155171
"1");
172+
defineGRPCView(views);
156173
return views.build();
157174
}
158175

@@ -183,4 +200,26 @@ private static void defineView(
183200
.build();
184201
viewMap.put(selector, view);
185202
}
203+
204+
private static void defineGRPCView(ImmutableMap.Builder<InstrumentSelector, View> viewMap) {
205+
for (String metric : BuiltInMetricsConstant.GRPC_METRICS_TO_ENABLE) {
206+
InstrumentSelector selector =
207+
InstrumentSelector.builder()
208+
.setName(metric)
209+
.setMeterName(BuiltInMetricsConstant.GRPC_METER_NAME)
210+
.build();
211+
Set<String> attributesFilter =
212+
BuiltInMetricsConstant.COMMON_ATTRIBUTES.stream()
213+
.map(AttributeKey::getKey)
214+
.collect(Collectors.toSet());
215+
attributesFilter.addAll(BuiltInMetricsConstant.GRPC_LB_RLS_ATTRIBUTES);
216+
217+
View view =
218+
View.builder()
219+
.setName(BuiltInMetricsConstant.METER_NAME + '/' + metric.replace(".", "/"))
220+
.setAttributeFilter(attributesFilter)
221+
.build();
222+
viewMap.put(selector, view);
223+
}
224+
}
186225
}

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsProvider.java

Lines changed: 50 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,19 +21,28 @@
2121
import static com.google.cloud.spanner.BuiltInMetricsConstant.CLIENT_NAME_KEY;
2222
import static com.google.cloud.spanner.BuiltInMetricsConstant.CLIENT_UID_KEY;
2323
import static com.google.cloud.spanner.BuiltInMetricsConstant.INSTANCE_CONFIG_ID_KEY;
24+
import static com.google.cloud.spanner.BuiltInMetricsConstant.INSTANCE_ID_KEY;
2425
import static com.google.cloud.spanner.BuiltInMetricsConstant.LOCATION_ID_KEY;
2526
import static com.google.cloud.spanner.BuiltInMetricsConstant.PROJECT_ID_KEY;
2627

28+
import com.google.api.core.ApiFunction;
29+
import com.google.api.gax.core.GaxProperties;
30+
import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider;
2731
import com.google.auth.Credentials;
2832
import com.google.cloud.opentelemetry.detection.AttributeKeys;
2933
import com.google.cloud.opentelemetry.detection.DetectedPlatform;
3034
import com.google.cloud.opentelemetry.detection.GCPPlatformDetector;
3135
import com.google.common.hash.HashFunction;
3236
import com.google.common.hash.Hashing;
37+
import io.grpc.ManagedChannelBuilder;
38+
import io.grpc.opentelemetry.GrpcOpenTelemetry;
3339
import io.opentelemetry.api.OpenTelemetry;
40+
import io.opentelemetry.api.common.Attributes;
41+
import io.opentelemetry.api.common.AttributesBuilder;
3442
import io.opentelemetry.sdk.OpenTelemetrySdk;
3543
import io.opentelemetry.sdk.metrics.SdkMeterProvider;
3644
import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder;
45+
import io.opentelemetry.sdk.resources.Resource;
3746
import java.io.IOException;
3847
import java.lang.management.ManagementFactory;
3948
import java.lang.reflect.Method;
@@ -66,6 +75,7 @@ OpenTelemetry getOrCreateOpenTelemetry(
6675
BuiltInMetricsView.registerBuiltinMetrics(
6776
SpannerCloudMonitoringExporter.create(projectId, credentials, monitoringHost),
6877
sdkMeterProviderBuilder);
78+
sdkMeterProviderBuilder.setResource(Resource.create(createResourceAttributes(projectId)));
6979
SdkMeterProvider sdkMeterProvider = sdkMeterProviderBuilder.build();
7080
this.openTelemetry = OpenTelemetrySdk.builder().setMeterProvider(sdkMeterProvider).build();
7181
Runtime.getRuntime().addShutdownHook(new Thread(sdkMeterProvider::close));
@@ -80,15 +90,47 @@ OpenTelemetry getOrCreateOpenTelemetry(
8090
}
8191
}
8292

83-
Map<String, String> createClientAttributes(String projectId, String client_name) {
93+
void enableGrpcMetrics(
94+
InstantiatingGrpcChannelProvider.Builder channelProviderBuilder,
95+
String projectId,
96+
@Nullable Credentials credentials,
97+
@Nullable String monitoringHost) {
98+
GrpcOpenTelemetry grpcOpenTelemetry =
99+
GrpcOpenTelemetry.newBuilder()
100+
.sdk(this.getOrCreateOpenTelemetry(projectId, credentials, monitoringHost))
101+
.enableMetrics(BuiltInMetricsConstant.GRPC_METRICS_TO_ENABLE)
102+
// Disable gRPCs default metrics as they are not needed for Spanner.
103+
.disableMetrics(BuiltInMetricsConstant.GRPC_METRICS_ENABLED_BY_DEFAULT)
104+
.build();
105+
ApiFunction<ManagedChannelBuilder, ManagedChannelBuilder> channelConfigurator =
106+
channelProviderBuilder.getChannelConfigurator();
107+
channelProviderBuilder.setChannelConfigurator(
108+
b -> {
109+
grpcOpenTelemetry.configureChannelBuilder(b);
110+
if (channelConfigurator != null) {
111+
return channelConfigurator.apply(b);
112+
}
113+
return b;
114+
});
115+
}
116+
117+
Attributes createResourceAttributes(String projectId) {
118+
AttributesBuilder attributesBuilder =
119+
Attributes.builder()
120+
.put(PROJECT_ID_KEY.getKey(), projectId)
121+
.put(INSTANCE_CONFIG_ID_KEY.getKey(), "unknown")
122+
.put(CLIENT_HASH_KEY.getKey(), generateClientHash(getDefaultTaskValue()))
123+
.put(INSTANCE_ID_KEY.getKey(), "unknown")
124+
.put(LOCATION_ID_KEY.getKey(), detectClientLocation());
125+
126+
return attributesBuilder.build();
127+
}
128+
129+
Map<String, String> createClientAttributes() {
84130
Map<String, String> clientAttributes = new HashMap<>();
85-
clientAttributes.put(LOCATION_ID_KEY.getKey(), detectClientLocation());
86-
clientAttributes.put(PROJECT_ID_KEY.getKey(), projectId);
87-
clientAttributes.put(INSTANCE_CONFIG_ID_KEY.getKey(), "unknown");
88-
clientAttributes.put(CLIENT_NAME_KEY.getKey(), client_name);
89-
String clientUid = getDefaultTaskValue();
90-
clientAttributes.put(CLIENT_UID_KEY.getKey(), clientUid);
91-
clientAttributes.put(CLIENT_HASH_KEY.getKey(), generateClientHash(clientUid));
131+
clientAttributes.put(
132+
CLIENT_NAME_KEY.getKey(), "spanner-java/" + GaxProperties.getLibraryVersion(getClass()));
133+
clientAttributes.put(CLIENT_UID_KEY.getKey(), getDefaultTaskValue());
92134
return clientAttributes;
93135
}
94136

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java

Lines changed: 8 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,6 @@
1616

1717
package com.google.cloud.spanner;
1818

19-
import static com.google.cloud.spanner.BuiltInMetricsConstant.SPANNER_METRICS;
20-
2119
import com.google.api.core.ApiFuture;
2220
import com.google.api.core.ApiFutureCallback;
2321
import com.google.api.core.ApiFutures;
@@ -39,8 +37,8 @@
3937
import io.opentelemetry.sdk.metrics.InstrumentType;
4038
import io.opentelemetry.sdk.metrics.data.AggregationTemporality;
4139
import io.opentelemetry.sdk.metrics.data.MetricData;
42-
import io.opentelemetry.sdk.metrics.data.PointData;
4340
import io.opentelemetry.sdk.metrics.export.MetricExporter;
41+
import io.opentelemetry.sdk.resources.Resource;
4442
import java.io.IOException;
4543
import java.time.Duration;
4644
import java.util.ArrayList;
@@ -114,27 +112,19 @@ public CompletableResultCode export(@Nonnull Collection<MetricData> collection)
114112

115113
/** Export client built in metrics */
116114
private CompletableResultCode exportSpannerClientMetrics(Collection<MetricData> collection) {
117-
// Filter spanner metrics. Only include metrics that contain a project and instance ID.
118-
List<MetricData> spannerMetricData =
119-
collection.stream()
120-
.filter(md -> SPANNER_METRICS.contains(md.getName()))
121-
.collect(Collectors.toList());
115+
// Filter spanner metrics. Only include metrics that contain a valid project.
116+
List<MetricData> spannerMetricData = collection.stream().collect(Collectors.toList());
122117

123118
// Log warnings for metrics that will be skipped.
124119
boolean mustFilter = false;
125120
if (spannerMetricData.stream()
126-
.flatMap(metricData -> metricData.getData().getPoints().stream())
121+
.map(metricData -> metricData.getResource())
127122
.anyMatch(this::shouldSkipPointDataDueToProjectId)) {
128123
logger.log(
129124
Level.WARNING, "Some metric data contain a different projectId. These will be skipped.");
130125
mustFilter = true;
131126
}
132-
if (spannerMetricData.stream()
133-
.flatMap(metricData -> metricData.getData().getPoints().stream())
134-
.anyMatch(this::shouldSkipPointDataDueToMissingInstanceId)) {
135-
logger.log(Level.WARNING, "Some metric data miss instanceId. These will be skipped.");
136-
mustFilter = true;
137-
}
127+
138128
if (mustFilter) {
139129
spannerMetricData =
140130
spannerMetricData.stream()
@@ -198,19 +188,11 @@ public void onSuccess(List<Empty> empty) {
198188
}
199189

200190
private boolean shouldSkipMetricData(MetricData metricData) {
201-
return metricData.getData().getPoints().stream()
202-
.anyMatch(
203-
pd ->
204-
shouldSkipPointDataDueToProjectId(pd)
205-
|| shouldSkipPointDataDueToMissingInstanceId(pd));
206-
}
207-
208-
private boolean shouldSkipPointDataDueToProjectId(PointData pointData) {
209-
return !spannerProjectId.equals(SpannerCloudMonitoringExporterUtils.getProjectId(pointData));
191+
return shouldSkipPointDataDueToProjectId(metricData.getResource());
210192
}
211193

212-
private boolean shouldSkipPointDataDueToMissingInstanceId(PointData pointData) {
213-
return SpannerCloudMonitoringExporterUtils.getInstanceId(pointData) == null;
194+
private boolean shouldSkipPointDataDueToProjectId(Resource resource) {
195+
return !spannerProjectId.equals(SpannerCloudMonitoringExporterUtils.getProjectId(resource));
214196
}
215197

216198
boolean lastExportSkippedData() {

0 commit comments

Comments
 (0)