diff --git a/google-cloud-spanner/clirr-ignored-differences.xml b/google-cloud-spanner/clirr-ignored-differences.xml index b6b19d1d577..d6d36b0e147 100644 --- a/google-cloud-spanner/clirr-ignored-differences.xml +++ b/google-cloud-spanner/clirr-ignored-differences.xml @@ -751,6 +751,13 @@ boolean isEnableBuiltInMetrics() + + + 7012 + com/google/cloud/spanner/SpannerOptions$SpannerEnvironment + boolean isEnableGRPCBuiltInMetrics() + + 7012 @@ -807,7 +814,7 @@ com/google/cloud/spanner/connection/Connection boolean isKeepTransactionAlive() - + 7012 @@ -839,7 +846,7 @@ com/google/cloud/spanner/connection/Connection boolean isAutoBatchDmlUpdateCountVerification() - + 7012 @@ -863,7 +870,7 @@ com/google/cloud/spanner/connection/Connection java.lang.Object runTransaction(com.google.cloud.spanner.connection.Connection$TransactionCallable) - + 7012 diff --git a/google-cloud-spanner/pom.xml b/google-cloud-spanner/pom.xml index eed3735b857..909b96698c5 100644 --- a/google-cloud-spanner/pom.xml +++ b/google-cloud-spanner/pom.xml @@ -191,6 +191,10 @@ io.grpc grpc-stub + + io.grpc + grpc-opentelemetry + com.google.api api-common diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java index 4adf53d7e40..050484ae66e 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java @@ -26,6 +26,7 @@ import io.opentelemetry.sdk.metrics.InstrumentSelector; import io.opentelemetry.sdk.metrics.InstrumentType; import io.opentelemetry.sdk.metrics.View; +import java.util.Collection; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -36,6 +37,7 @@ public class BuiltInMetricsConstant { public static final String METER_NAME = "spanner.googleapis.com/internal/client"; public static final String GAX_METER_NAME = OpenTelemetryMetricsRecorder.GAX_METER_NAME; static final String SPANNER_METER_NAME = "spanner-java"; + static final String GRPC_METER_NAME = "grpc-java"; static final String GFE_LATENCIES_NAME = "gfe_latencies"; static final String OPERATION_LATENCIES_NAME = "operation_latencies"; static final String ATTEMPT_LATENCIES_NAME = "attempt_latencies"; @@ -55,6 +57,14 @@ public class BuiltInMetricsConstant { .map(m -> METER_NAME + '/' + m) .collect(Collectors.toSet()); + static final Collection GRPC_METRICS_TO_ENABLE = + ImmutableList.of( + "grpc.lb.rls.default_target_picks", + "grpc.lb.rls.target_picks", + "grpc.xds_client.server_failure", + "grpc.xds_client.resource_updates_invalid", + "grpc.xds_client.resource_updates_valid"); + public static final String SPANNER_RESOURCE_TYPE = "spanner_instance_client"; public static final AttributeKey PROJECT_ID_KEY = AttributeKey.stringKey("project_id"); @@ -66,12 +76,7 @@ public class BuiltInMetricsConstant { // These metric labels will be promoted to the spanner monitored resource fields public static final Set> SPANNER_PROMOTED_RESOURCE_LABELS = - ImmutableSet.of( - PROJECT_ID_KEY, - INSTANCE_ID_KEY, - INSTANCE_CONFIG_ID_KEY, - LOCATION_ID_KEY, - CLIENT_HASH_KEY); + ImmutableSet.of(INSTANCE_ID_KEY); public static final AttributeKey DATABASE_KEY = AttributeKey.stringKey("database"); public static final AttributeKey CLIENT_UID_KEY = AttributeKey.stringKey("client_uid"); @@ -102,6 +107,9 @@ public class BuiltInMetricsConstant { DIRECT_PATH_ENABLED_KEY, DIRECT_PATH_USED_KEY); + static final Set GRPC_LB_RLS_ATTRIBUTES = + ImmutableSet.of("grpc.lb.rls.data_plane_target", "grpc.lb.pick_result"); + static Aggregation AGGREGATION_WITH_MILLIS_HISTOGRAM = Aggregation.explicitBucketHistogram( ImmutableList.of( @@ -111,6 +119,14 @@ public class BuiltInMetricsConstant { 10000.0, 20000.0, 50000.0, 100000.0, 200000.0, 400000.0, 800000.0, 1600000.0, 3200000.0)); + static final Collection GRPC_METRICS_ENABLED_BY_DEFAULT = + ImmutableList.of( + "grpc.client.attempt.sent_total_compressed_message_size", + "grpc.client.attempt.rcvd_total_compressed_message_size", + "grpc.client.attempt.started", + "grpc.client.attempt.duration", + "grpc.client.call.duration"); + static Map getAllViews() { ImmutableMap.Builder views = ImmutableMap.builder(); defineView( @@ -153,6 +169,7 @@ static Map getAllViews() { Aggregation.sum(), InstrumentType.COUNTER, "1"); + defineGRPCView(views); return views.build(); } @@ -183,4 +200,26 @@ private static void defineView( .build(); viewMap.put(selector, view); } + + private static void defineGRPCView(ImmutableMap.Builder viewMap) { + for (String metric : BuiltInMetricsConstant.GRPC_METRICS_TO_ENABLE) { + InstrumentSelector selector = + InstrumentSelector.builder() + .setName(metric) + .setMeterName(BuiltInMetricsConstant.GRPC_METER_NAME) + .build(); + Set attributesFilter = + BuiltInMetricsConstant.COMMON_ATTRIBUTES.stream() + .map(AttributeKey::getKey) + .collect(Collectors.toSet()); + attributesFilter.addAll(BuiltInMetricsConstant.GRPC_LB_RLS_ATTRIBUTES); + + View view = + View.builder() + .setName(BuiltInMetricsConstant.METER_NAME + '/' + metric.replace(".", "/")) + .setAttributeFilter(attributesFilter) + .build(); + viewMap.put(selector, view); + } + } } diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsProvider.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsProvider.java index f624f310f77..888eff90b58 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsProvider.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsProvider.java @@ -21,19 +21,28 @@ import static com.google.cloud.spanner.BuiltInMetricsConstant.CLIENT_NAME_KEY; import static com.google.cloud.spanner.BuiltInMetricsConstant.CLIENT_UID_KEY; import static com.google.cloud.spanner.BuiltInMetricsConstant.INSTANCE_CONFIG_ID_KEY; +import static com.google.cloud.spanner.BuiltInMetricsConstant.INSTANCE_ID_KEY; import static com.google.cloud.spanner.BuiltInMetricsConstant.LOCATION_ID_KEY; import static com.google.cloud.spanner.BuiltInMetricsConstant.PROJECT_ID_KEY; +import com.google.api.core.ApiFunction; +import com.google.api.gax.core.GaxProperties; +import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; import com.google.auth.Credentials; import com.google.cloud.opentelemetry.detection.AttributeKeys; import com.google.cloud.opentelemetry.detection.DetectedPlatform; import com.google.cloud.opentelemetry.detection.GCPPlatformDetector; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; +import io.grpc.ManagedChannelBuilder; +import io.grpc.opentelemetry.GrpcOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.metrics.SdkMeterProvider; import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; +import io.opentelemetry.sdk.resources.Resource; import java.io.IOException; import java.lang.management.ManagementFactory; import java.lang.reflect.Method; @@ -66,6 +75,7 @@ OpenTelemetry getOrCreateOpenTelemetry( BuiltInMetricsView.registerBuiltinMetrics( SpannerCloudMonitoringExporter.create(projectId, credentials, monitoringHost), sdkMeterProviderBuilder); + sdkMeterProviderBuilder.setResource(Resource.create(createResourceAttributes(projectId))); SdkMeterProvider sdkMeterProvider = sdkMeterProviderBuilder.build(); this.openTelemetry = OpenTelemetrySdk.builder().setMeterProvider(sdkMeterProvider).build(); Runtime.getRuntime().addShutdownHook(new Thread(sdkMeterProvider::close)); @@ -80,15 +90,47 @@ OpenTelemetry getOrCreateOpenTelemetry( } } - Map createClientAttributes(String projectId, String client_name) { + void enableGrpcMetrics( + InstantiatingGrpcChannelProvider.Builder channelProviderBuilder, + String projectId, + @Nullable Credentials credentials, + @Nullable String monitoringHost) { + GrpcOpenTelemetry grpcOpenTelemetry = + GrpcOpenTelemetry.newBuilder() + .sdk(this.getOrCreateOpenTelemetry(projectId, credentials, monitoringHost)) + .enableMetrics(BuiltInMetricsConstant.GRPC_METRICS_TO_ENABLE) + // Disable gRPCs default metrics as they are not needed for Spanner. + .disableMetrics(BuiltInMetricsConstant.GRPC_METRICS_ENABLED_BY_DEFAULT) + .build(); + ApiFunction channelConfigurator = + channelProviderBuilder.getChannelConfigurator(); + channelProviderBuilder.setChannelConfigurator( + b -> { + grpcOpenTelemetry.configureChannelBuilder(b); + if (channelConfigurator != null) { + return channelConfigurator.apply(b); + } + return b; + }); + } + + Attributes createResourceAttributes(String projectId) { + AttributesBuilder attributesBuilder = + Attributes.builder() + .put(PROJECT_ID_KEY.getKey(), projectId) + .put(INSTANCE_CONFIG_ID_KEY.getKey(), "unknown") + .put(CLIENT_HASH_KEY.getKey(), generateClientHash(getDefaultTaskValue())) + .put(INSTANCE_ID_KEY.getKey(), "unknown") + .put(LOCATION_ID_KEY.getKey(), detectClientLocation()); + + return attributesBuilder.build(); + } + + Map createClientAttributes() { Map clientAttributes = new HashMap<>(); - clientAttributes.put(LOCATION_ID_KEY.getKey(), detectClientLocation()); - clientAttributes.put(PROJECT_ID_KEY.getKey(), projectId); - clientAttributes.put(INSTANCE_CONFIG_ID_KEY.getKey(), "unknown"); - clientAttributes.put(CLIENT_NAME_KEY.getKey(), client_name); - String clientUid = getDefaultTaskValue(); - clientAttributes.put(CLIENT_UID_KEY.getKey(), clientUid); - clientAttributes.put(CLIENT_HASH_KEY.getKey(), generateClientHash(clientUid)); + clientAttributes.put( + CLIENT_NAME_KEY.getKey(), "spanner-java/" + GaxProperties.getLibraryVersion(getClass())); + clientAttributes.put(CLIENT_UID_KEY.getKey(), getDefaultTaskValue()); return clientAttributes; } diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java index f9c91b91833..35503fff337 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java @@ -16,8 +16,6 @@ package com.google.cloud.spanner; -import static com.google.cloud.spanner.BuiltInMetricsConstant.SPANNER_METRICS; - import com.google.api.core.ApiFuture; import com.google.api.core.ApiFutureCallback; import com.google.api.core.ApiFutures; @@ -39,8 +37,8 @@ import io.opentelemetry.sdk.metrics.InstrumentType; import io.opentelemetry.sdk.metrics.data.AggregationTemporality; import io.opentelemetry.sdk.metrics.data.MetricData; -import io.opentelemetry.sdk.metrics.data.PointData; import io.opentelemetry.sdk.metrics.export.MetricExporter; +import io.opentelemetry.sdk.resources.Resource; import java.io.IOException; import java.time.Duration; import java.util.ArrayList; @@ -114,27 +112,19 @@ public CompletableResultCode export(@Nonnull Collection collection) /** Export client built in metrics */ private CompletableResultCode exportSpannerClientMetrics(Collection collection) { - // Filter spanner metrics. Only include metrics that contain a project and instance ID. - List spannerMetricData = - collection.stream() - .filter(md -> SPANNER_METRICS.contains(md.getName())) - .collect(Collectors.toList()); + // Filter spanner metrics. Only include metrics that contain a valid project. + List spannerMetricData = collection.stream().collect(Collectors.toList()); // Log warnings for metrics that will be skipped. boolean mustFilter = false; if (spannerMetricData.stream() - .flatMap(metricData -> metricData.getData().getPoints().stream()) + .map(metricData -> metricData.getResource()) .anyMatch(this::shouldSkipPointDataDueToProjectId)) { logger.log( Level.WARNING, "Some metric data contain a different projectId. These will be skipped."); mustFilter = true; } - if (spannerMetricData.stream() - .flatMap(metricData -> metricData.getData().getPoints().stream()) - .anyMatch(this::shouldSkipPointDataDueToMissingInstanceId)) { - logger.log(Level.WARNING, "Some metric data miss instanceId. These will be skipped."); - mustFilter = true; - } + if (mustFilter) { spannerMetricData = spannerMetricData.stream() @@ -198,19 +188,11 @@ public void onSuccess(List empty) { } private boolean shouldSkipMetricData(MetricData metricData) { - return metricData.getData().getPoints().stream() - .anyMatch( - pd -> - shouldSkipPointDataDueToProjectId(pd) - || shouldSkipPointDataDueToMissingInstanceId(pd)); - } - - private boolean shouldSkipPointDataDueToProjectId(PointData pointData) { - return !spannerProjectId.equals(SpannerCloudMonitoringExporterUtils.getProjectId(pointData)); + return shouldSkipPointDataDueToProjectId(metricData.getResource()); } - private boolean shouldSkipPointDataDueToMissingInstanceId(PointData pointData) { - return SpannerCloudMonitoringExporterUtils.getInstanceId(pointData) == null; + private boolean shouldSkipPointDataDueToProjectId(Resource resource) { + return !spannerProjectId.equals(SpannerCloudMonitoringExporterUtils.getProjectId(resource)); } boolean lastExportSkippedData() { diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporterUtils.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporterUtils.java index 620430b87df..f67621db963 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporterUtils.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporterUtils.java @@ -23,7 +23,7 @@ import static com.google.api.MetricDescriptor.ValueType.DOUBLE; import static com.google.api.MetricDescriptor.ValueType.INT64; import static com.google.cloud.spanner.BuiltInMetricsConstant.GAX_METER_NAME; -import static com.google.cloud.spanner.BuiltInMetricsConstant.INSTANCE_ID_KEY; +import static com.google.cloud.spanner.BuiltInMetricsConstant.GRPC_METER_NAME; import static com.google.cloud.spanner.BuiltInMetricsConstant.PROJECT_ID_KEY; import static com.google.cloud.spanner.BuiltInMetricsConstant.SPANNER_METER_NAME; import static com.google.cloud.spanner.BuiltInMetricsConstant.SPANNER_PROMOTED_RESOURCE_LABELS; @@ -52,6 +52,7 @@ import io.opentelemetry.sdk.metrics.data.MetricDataType; import io.opentelemetry.sdk.metrics.data.PointData; import io.opentelemetry.sdk.metrics.data.SumData; +import io.opentelemetry.sdk.resources.Resource; import java.util.ArrayList; import java.util.List; import java.util.logging.Level; @@ -64,34 +65,45 @@ class SpannerCloudMonitoringExporterUtils { private SpannerCloudMonitoringExporterUtils() {} - static String getProjectId(PointData pointData) { - return pointData.getAttributes().get(PROJECT_ID_KEY); - } - - static String getInstanceId(PointData pointData) { - return pointData.getAttributes().get(INSTANCE_ID_KEY); + static String getProjectId(Resource resource) { + return resource.getAttributes().get(PROJECT_ID_KEY); } static List convertToSpannerTimeSeries(List collection) { List allTimeSeries = new ArrayList<>(); for (MetricData metricData : collection) { - // Get metrics data from GAX library and Spanner library + // Get metrics data from GAX library, GRPC library and Spanner library if (!(metricData.getInstrumentationScopeInfo().getName().equals(GAX_METER_NAME) - || metricData.getInstrumentationScopeInfo().getName().equals(SPANNER_METER_NAME))) { + || metricData.getInstrumentationScopeInfo().getName().equals(SPANNER_METER_NAME) + || metricData.getInstrumentationScopeInfo().getName().equals(GRPC_METER_NAME))) { // Filter out metric data for instruments that are not part of the spanner metrics list continue; } + + // Create MonitoredResource Builder + MonitoredResource.Builder monitoredResourceBuilder = + MonitoredResource.newBuilder().setType(SPANNER_RESOURCE_TYPE); + + Attributes resourceAttributes = metricData.getResource().getAttributes(); + for (AttributeKey key : resourceAttributes.asMap().keySet()) { + monitoredResourceBuilder.putLabels( + key.getKey(), String.valueOf(resourceAttributes.get(key))); + } + metricData.getData().getPoints().stream() - .map(pointData -> convertPointToSpannerTimeSeries(metricData, pointData)) + .map( + pointData -> + convertPointToSpannerTimeSeries(metricData, pointData, monitoredResourceBuilder)) .forEach(allTimeSeries::add); } - return allTimeSeries; } private static TimeSeries convertPointToSpannerTimeSeries( - MetricData metricData, PointData pointData) { + MetricData metricData, + PointData pointData, + MonitoredResource.Builder monitoredResourceBuilder) { TimeSeries.Builder builder = TimeSeries.newBuilder() .setMetricKind(convertMetricKind(metricData)) @@ -99,17 +111,21 @@ private static TimeSeries convertPointToSpannerTimeSeries( Metric.Builder metricBuilder = Metric.newBuilder().setType(metricData.getName()); Attributes attributes = pointData.getAttributes(); - MonitoredResource.Builder monitoredResourceBuilder = - MonitoredResource.newBuilder().setType(SPANNER_RESOURCE_TYPE); for (AttributeKey key : attributes.asMap().keySet()) { if (SPANNER_PROMOTED_RESOURCE_LABELS.contains(key)) { monitoredResourceBuilder.putLabels(key.getKey(), String.valueOf(attributes.get(key))); } else { - metricBuilder.putLabels(key.getKey(), String.valueOf(attributes.get(key))); + // Replace metric label names by converting "." to "_" since Cloud Monitoring does not + // support labels containing "." + metricBuilder.putLabels( + key.getKey().replace(".", "_"), String.valueOf(attributes.get(key))); } } + // Add common labels like "client_name" and "client_uid" for all the exported metrics. + metricBuilder.putAllLabels(BuiltInMetricsProvider.INSTANCE.createClientAttributes()); + builder.setResource(monitoredResourceBuilder.build()); builder.setMetric(metricBuilder.build()); diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java index 695e156dfc3..8a9c5050d13 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java @@ -27,6 +27,7 @@ import com.google.api.gax.core.GaxProperties; import com.google.api.gax.grpc.GrpcCallContext; import com.google.api.gax.grpc.GrpcInterceptorProvider; +import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; import com.google.api.gax.longrunning.OperationTimedPollAlgorithm; import com.google.api.gax.retrying.RetrySettings; import com.google.api.gax.rpc.ApiCallContext; @@ -848,6 +849,10 @@ default boolean isEnableBuiltInMetrics() { return true; } + default boolean isEnableGRPCBuiltInMetrics() { + return false; + } + default boolean isEnableEndToEndTracing() { return false; } @@ -878,6 +883,8 @@ private static class SpannerEnvironmentImpl implements SpannerEnvironment { private static final String SPANNER_ENABLE_END_TO_END_TRACING = "SPANNER_ENABLE_END_TO_END_TRACING"; private static final String SPANNER_DISABLE_BUILTIN_METRICS = "SPANNER_DISABLE_BUILTIN_METRICS"; + private static final String SPANNER_DISABLE_DIRECT_ACCESS_GRPC_BUILTIN_METRICS = + "SPANNER_DISABLE_DIRECT_ACCESS_GRPC_BUILTIN_METRICS"; private static final String SPANNER_MONITORING_HOST = "SPANNER_MONITORING_HOST"; private SpannerEnvironmentImpl() {} @@ -910,6 +917,12 @@ public boolean isEnableBuiltInMetrics() { return !Boolean.parseBoolean(System.getenv(SPANNER_DISABLE_BUILTIN_METRICS)); } + @Override + public boolean isEnableGRPCBuiltInMetrics() { + return "false" + .equalsIgnoreCase(System.getenv(SPANNER_DISABLE_DIRECT_ACCESS_GRPC_BUILTIN_METRICS)); + } + @Override public boolean isEnableEndToEndTracing() { return Boolean.parseBoolean(System.getenv(SPANNER_ENABLE_END_TO_END_TRACING)); @@ -1971,6 +1984,13 @@ public ApiTracerFactory getApiTracerFactory() { return createApiTracerFactory(false, false); } + public void enablegRPCMetrics(InstantiatingGrpcChannelProvider.Builder channelProviderBuilder) { + if (SpannerOptions.environment.isEnableGRPCBuiltInMetrics()) { + this.builtInMetricsProvider.enableGrpcMetrics( + channelProviderBuilder, this.getProjectId(), getCredentials(), this.monitoringHost); + } + } + public ApiTracerFactory getApiTracerFactory(boolean isAdminClient, boolean isEmulatorEnabled) { return createApiTracerFactory(isAdminClient, isEmulatorEnabled); } @@ -2018,8 +2038,7 @@ private ApiTracerFactory createMetricsApiTracerFactory() { return openTelemetry != null ? new BuiltInMetricsTracerFactory( new BuiltInMetricsRecorder(openTelemetry, BuiltInMetricsConstant.METER_NAME), - builtInMetricsProvider.createClientAttributes( - this.getProjectId(), "spanner-java/" + GaxProperties.getLibraryVersion(getClass()))) + new HashMap<>()) : null; } diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java index 0f51c9544f7..2a0ac1839dd 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java @@ -373,6 +373,9 @@ public GapicSpannerRpc(final SpannerOptions options) { defaultChannelProviderBuilder.setAttemptDirectPath(true); defaultChannelProviderBuilder.setAttemptDirectPathXds(); } + + options.enablegRPCMetrics(defaultChannelProviderBuilder); + if (options.isUseVirtualThreads()) { ExecutorService executor = tryCreateVirtualThreadPerTaskExecutor("spanner-virtual-grpc-executor"); diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java index f0c13b0f389..d2db2e4bca0 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java @@ -91,18 +91,12 @@ public static void setup() { String client_name = "spanner-java/"; openTelemetry = OpenTelemetrySdk.builder().setMeterProvider(meterProvider.build()).build(); - attributes = provider.createClientAttributes("test-project", client_name); + attributes = provider.createClientAttributes(); expectedCommonBaseAttributes = Attributes.builder() - .put(BuiltInMetricsConstant.PROJECT_ID_KEY, "test-project") - .put(BuiltInMetricsConstant.INSTANCE_CONFIG_ID_KEY, "unknown") - .put( - BuiltInMetricsConstant.LOCATION_ID_KEY, - BuiltInMetricsProvider.detectClientLocation()) .put(BuiltInMetricsConstant.CLIENT_NAME_KEY, client_name) .put(BuiltInMetricsConstant.CLIENT_UID_KEY, attributes.get("client_uid")) - .put(BuiltInMetricsConstant.CLIENT_HASH_KEY, attributes.get("client_hash")) .put(BuiltInMetricsConstant.INSTANCE_ID_KEY, "i") .put(BuiltInMetricsConstant.DATABASE_KEY, "d") .put(BuiltInMetricsConstant.DIRECT_PATH_ENABLED_KEY, "false") diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SpannerCloudMonitoringExporterTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SpannerCloudMonitoringExporterTest.java index f9a6e9df9a6..84a8cf4460c 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SpannerCloudMonitoringExporterTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SpannerCloudMonitoringExporterTest.java @@ -16,7 +16,6 @@ package com.google.cloud.spanner; -import static com.google.cloud.spanner.BuiltInMetricsConstant.ATTEMPT_COUNT_NAME; import static com.google.cloud.spanner.BuiltInMetricsConstant.CLIENT_HASH_KEY; import static com.google.cloud.spanner.BuiltInMetricsConstant.CLIENT_NAME_KEY; import static com.google.cloud.spanner.BuiltInMetricsConstant.CLIENT_UID_KEY; @@ -32,7 +31,6 @@ import static com.google.cloud.spanner.BuiltInMetricsConstant.PROJECT_ID_KEY; import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -47,7 +45,6 @@ import com.google.monitoring.v3.TimeSeries; import com.google.protobuf.Empty; import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.sdk.common.CompletableResultCode; import io.opentelemetry.sdk.common.InstrumentationScopeInfo; import io.opentelemetry.sdk.metrics.InstrumentType; import io.opentelemetry.sdk.metrics.data.AggregationTemporality; @@ -81,7 +78,7 @@ public class SpannerCloudMonitoringExporterTest { private static final String instanceId = "fake-instance"; private static final String locationId = "global"; private static final String databaseId = "fake-database"; - private static final String clientName = "spanner-java"; + private static final String clientName = "spanner-java/"; private static final String clientHash = "spanner-test"; private static final String instanceConfigId = "fake-instance-config-id"; @@ -93,28 +90,38 @@ public class SpannerCloudMonitoringExporterTest { private SpannerCloudMonitoringExporter exporter; private Attributes attributes; + + private Attributes resourceAttributes; private Resource resource; private InstrumentationScopeInfo scope; + private String client_uid; + @Before public void setUp() { fakeMetricServiceClient = new FakeMetricServiceClient(mockMetricServiceStub); exporter = new SpannerCloudMonitoringExporter(projectId, fakeMetricServiceClient); + this.client_uid = BuiltInMetricsProvider.INSTANCE.createClientAttributes().get("client_uid"); + attributes = Attributes.builder() - .put(PROJECT_ID_KEY, projectId) .put(INSTANCE_ID_KEY, instanceId) - .put(LOCATION_ID_KEY, locationId) - .put(INSTANCE_CONFIG_ID_KEY, instanceConfigId) .put(DATABASE_KEY, databaseId) .put(CLIENT_NAME_KEY, clientName) - .put(CLIENT_HASH_KEY, clientHash) + .put(CLIENT_UID_KEY, this.client_uid) .put(String.valueOf(DIRECT_PATH_ENABLED_KEY), true) .put(String.valueOf(DIRECT_PATH_USED_KEY), true) .build(); - resource = Resource.create(Attributes.empty()); + resourceAttributes = + Attributes.builder() + .put(PROJECT_ID_KEY, projectId) + .put(LOCATION_ID_KEY, locationId) + .put(CLIENT_HASH_KEY, clientHash) + .put(INSTANCE_CONFIG_ID_KEY, instanceConfigId) + .build(); + resource = Resource.create(resourceAttributes); scope = InstrumentationScopeInfo.create(GAX_METER_NAME); } @@ -177,8 +184,10 @@ public void testExportingSumData() { DIRECT_PATH_ENABLED_KEY.getKey(), "true", DIRECT_PATH_USED_KEY.getKey(), - "true"); - assertThat(timeSeries.getMetric().getLabelsMap()).hasSize(4); + "true", + CLIENT_UID_KEY.getKey(), + this.client_uid); + assertThat(timeSeries.getMetric().getLabelsMap()).hasSize(5); assertThat(timeSeries.getPoints(0).getValue().getInt64Value()).isEqualTo(fakeValue); assertThat(timeSeries.getPoints(0).getInterval().getStartTime().getNanos()) @@ -239,7 +248,7 @@ public void testExportingHistogramData() { INSTANCE_CONFIG_ID_KEY.getKey(), instanceConfigId, CLIENT_HASH_KEY.getKey(), clientHash); - assertThat(timeSeries.getMetric().getLabelsMap()).hasSize(4); + assertThat(timeSeries.getMetric().getLabelsMap()).hasSize(5); assertThat(timeSeries.getMetric().getLabelsMap()) .containsExactly( DATABASE_KEY.getKey(), @@ -249,7 +258,9 @@ public void testExportingHistogramData() { DIRECT_PATH_ENABLED_KEY.getKey(), "true", DIRECT_PATH_USED_KEY.getKey(), - "true"); + "true", + CLIENT_UID_KEY.getKey(), + this.client_uid); Distribution distribution = timeSeries.getPoints(0).getValue().getDistributionValue(); assertThat(distribution.getCount()).isEqualTo(3); @@ -274,11 +285,7 @@ public void testExportingSumDataInBatches() { Collection toExport = new ArrayList<>(); for (int i = 0; i < 250; i++) { LongPointData longPointData = - ImmutableLongPointData.create( - startEpoch, - endEpoch, - attributes.toBuilder().put(CLIENT_UID_KEY, "client_uid" + i).build(), - i); + ImmutableLongPointData.create(startEpoch, endEpoch, attributes, i); MetricData longData = ImmutableMetricData.createLongSum( @@ -331,7 +338,7 @@ public void testExportingSumDataInBatches() { DIRECT_PATH_USED_KEY.getKey(), "true", CLIENT_UID_KEY.getKey(), - "client_uid" + i); + this.client_uid); assertThat(timeSeries.getPoints(0).getValue().getInt64Value()).isEqualTo(i); assertThat(timeSeries.getPoints(0).getInterval().getStartTime().getNanos()) @@ -348,56 +355,6 @@ public void getAggregationTemporality() throws IOException { .isEqualTo(AggregationTemporality.CUMULATIVE); } - @Test - public void testSkipExportingDataIfMissingInstanceId() throws IOException { - Attributes attributesWithoutInstanceId = - Attributes.builder().putAll(attributes).remove(INSTANCE_ID_KEY).build(); - - SpannerCloudMonitoringExporter actualExporter = - SpannerCloudMonitoringExporter.create(projectId, null, null); - assertThat(actualExporter.getAggregationTemporality(InstrumentType.COUNTER)) - .isEqualTo(AggregationTemporality.CUMULATIVE); - ArgumentCaptor argumentCaptor = - ArgumentCaptor.forClass(CreateTimeSeriesRequest.class); - - UnaryCallable mockCallable = Mockito.mock(UnaryCallable.class); - Mockito.when(mockMetricServiceStub.createServiceTimeSeriesCallable()).thenReturn(mockCallable); - ApiFuture future = ApiFutures.immediateFuture(Empty.getDefaultInstance()); - Mockito.when(mockCallable.futureCall(argumentCaptor.capture())).thenReturn(future); - - long fakeValue = 11L; - - long startEpoch = 10; - long endEpoch = 15; - LongPointData longPointData = - ImmutableLongPointData.create(startEpoch, endEpoch, attributesWithoutInstanceId, fakeValue); - - MetricData operationLongData = - ImmutableMetricData.createLongSum( - resource, - scope, - "spanner.googleapis.com/internal/client/" + OPERATION_COUNT_NAME, - "description", - "1", - ImmutableSumData.create( - true, AggregationTemporality.CUMULATIVE, ImmutableList.of(longPointData))); - - MetricData attemptLongData = - ImmutableMetricData.createLongSum( - resource, - scope, - "spanner.googleapis.com/internal/client/" + ATTEMPT_COUNT_NAME, - "description", - "1", - ImmutableSumData.create( - true, AggregationTemporality.CUMULATIVE, ImmutableList.of(longPointData))); - - CompletableResultCode resultCode = - exporter.export(Arrays.asList(operationLongData, attemptLongData)); - assertTrue(resultCode.isSuccess()); - assertTrue(exporter.lastExportSkippedData()); - } - private static class FakeMetricServiceClient extends MetricServiceClient { protected FakeMetricServiceClient(MetricServiceStub stub) { diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITBuiltInMetricsTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITBuiltInMetricsTest.java index 5bf8e42ccb6..7eda6677764 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITBuiltInMetricsTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITBuiltInMetricsTest.java @@ -17,6 +17,7 @@ package com.google.cloud.spanner.it; import static com.google.common.truth.Truth.assertWithMessage; +import static org.junit.Assume.assumeFalse; import com.google.cloud.monitoring.v3.MetricServiceClient; import com.google.cloud.spanner.Database; @@ -24,6 +25,7 @@ import com.google.cloud.spanner.IntegrationTestEnv; import com.google.cloud.spanner.ParallelIntegrationTest; import com.google.cloud.spanner.Statement; +import com.google.cloud.spanner.testing.EmulatorSpannerHelper; import com.google.common.base.Stopwatch; import com.google.monitoring.v3.ListTimeSeriesRequest; import com.google.monitoring.v3.ListTimeSeriesResponse; @@ -34,9 +36,9 @@ import java.time.Duration; import java.time.Instant; import java.util.concurrent.TimeUnit; +import org.junit.After; import org.junit.BeforeClass; import org.junit.ClassRule; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -44,7 +46,6 @@ @Category(ParallelIntegrationTest.class) @RunWith(JUnit4.class) -@Ignore("Built-in Metrics are not GA'ed yet. Enable this test once the metrics are released") public class ITBuiltInMetricsTest { private static Database db; @@ -54,14 +55,26 @@ public class ITBuiltInMetricsTest { private static MetricServiceClient metricClient; + private static String[] METRICS = { + "operation_latencies", "attempt_latencies", "operation_count", "attempt_count", + }; + @BeforeClass public static void setUp() throws IOException { + assumeFalse("This test requires credentials", EmulatorSpannerHelper.isUsingEmulator()); metricClient = MetricServiceClient.create(); // Enable BuiltinMetrics when the metrics are GA'ed db = env.getTestHelper().createTestDatabase(); client = env.getTestHelper().getDatabaseClient(db); } + @After + public void tearDown() { + if (metricClient != null) { + metricClient.close(); + } + } + @Test public void testBuiltinMetricsWithDefaultOTEL() throws Exception { // This stopwatch is used for to limit fetching of metric data in verifyMetrics @@ -80,36 +93,36 @@ public void testBuiltinMetricsWithDefaultOTEL() throws Exception { .readWriteTransaction() .run(transaction -> transaction.executeQuery(Statement.of("Select 1"))); - String metricFilter = - String.format( - "metric.type=\"spanner.googleapis.com/client/%s\"" - + " AND resource.type=\"spanner_instance\"" - + " AND metric.labels.method=\"Spanner.Commit\"" - + " AND resource.labels.instance_id=\"%s\"" - + " AND metric.labels.database=\"%s\"", - "operation_latencies", - db.getId().getInstanceId().getInstance(), - db.getId().getDatabase()); - - ListTimeSeriesRequest.Builder requestBuilder = - ListTimeSeriesRequest.newBuilder() - .setName(name.toString()) - .setFilter(metricFilter) - .setInterval(interval) - .setView(ListTimeSeriesRequest.TimeSeriesView.FULL); - - ListTimeSeriesRequest request = requestBuilder.build(); - - ListTimeSeriesResponse response = metricClient.listTimeSeriesCallable().call(request); - while (response.getTimeSeriesCount() == 0 - && metricsPollingStopwatch.elapsed(TimeUnit.MINUTES) < 3) { - // Call listTimeSeries every minute - Thread.sleep(Duration.ofMinutes(1).toMillis()); - response = metricClient.listTimeSeriesCallable().call(request); + for (String metric : METRICS) { + String metricFilter = + String.format( + "metric.type=\"spanner.googleapis.com/client/%s\"" + + " AND resource.type=\"spanner_instance\"" + + " AND metric.labels.method=\"Spanner.Commit\"" + + " AND resource.labels.instance_id=\"%s\"" + + " AND metric.labels.database=\"%s\"", + metric, db.getId().getInstanceId().getInstance(), db.getId().getDatabase()); + + ListTimeSeriesRequest.Builder requestBuilder = + ListTimeSeriesRequest.newBuilder() + .setName(name.toString()) + .setFilter(metricFilter) + .setInterval(interval) + .setView(ListTimeSeriesRequest.TimeSeriesView.FULL); + + ListTimeSeriesRequest request = requestBuilder.build(); + + ListTimeSeriesResponse response = metricClient.listTimeSeriesCallable().call(request); + while (response.getTimeSeriesCount() == 0 + && metricsPollingStopwatch.elapsed(TimeUnit.MINUTES) < 3) { + // Call listTimeSeries every minute + Thread.sleep(Duration.ofMinutes(1).toMillis()); + response = metricClient.listTimeSeriesCallable().call(request); + } + + assertWithMessage("Metric" + metric + "didn't return any data.") + .that(response.getTimeSeriesCount()) + .isGreaterThan(0); } - - assertWithMessage("View operation_latencies didn't return any data.") - .that(response.getTimeSeriesCount()) - .isGreaterThan(0); } }