[CLN] Use GRPC for chroma-load OTEL (#4427)

## Description of changes

Remove sampling. Will leverage refinery for this instead.

## Test plan

These changes are untested. However, it is now using (mostly) the same
setup code as the other chroma services (which are working). So, we can
test in staging after this is merged.

Author: jasonvigil

rust/load/src/opentelemetry_config.rs

@@ -5,114 +5,41 @@
 
 use opentelemetry::global;
 use opentelemetry::trace::TracerProvider;
-use opentelemetry_otlp::{WithExportConfig, WithHttpConfig};
+use opentelemetry_otlp::WithExportConfig;
 use opentelemetry_sdk::propagation::TraceContextPropagator;
 use tracing_bunyan_formatter::BunyanFormattingLayer;
 use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Layer};
 
-#[derive(Clone, Debug, Default)]
-struct ChromaShouldSample;
-
-const BUSY_NS: opentelemetry::Key = opentelemetry::Key::from_static_str("busy_ns");
-const IDLE_NS: opentelemetry::Key = opentelemetry::Key::from_static_str("idle_ns");
-
-fn is_slow(attributes: &[opentelemetry::KeyValue]) -> bool {
-    let mut nanos = 0i64;
-    for attr in attributes {
-        if attr.key == BUSY_NS || attr.key == IDLE_NS {
-            if let opentelemetry::Value::I64(ns) = attr.value {
-                nanos += ns;
-            }
-        }
-    }
-    nanos > 20_000_000
-}
-
-impl opentelemetry_sdk::trace::ShouldSample for ChromaShouldSample {
-    fn should_sample(
-        &self,
-        _: Option<&opentelemetry::Context>,
-        _: opentelemetry::trace::TraceId,
-        name: &str,
-        _: &opentelemetry::trace::SpanKind,
-        attributes: &[opentelemetry::KeyValue],
-        _: &[opentelemetry::trace::Link],
-    ) -> opentelemetry::trace::SamplingResult {
-        // NOTE(rescrv):  THIS IS A HACK!  If you find yourself seriously extending it, it's time
-        // to investigate honeycomb's sampling capabilities.
-
-        // If the name is not get and not insert, or the request is slow, sample it.
-        // Otherwise, drop.
-        // This filters filters foyer calls in-process so they won't be overwhelming the tracing.
-        if (name != "get" && name != "insert") || is_slow(attributes) {
-            opentelemetry::trace::SamplingResult {
-                decision: opentelemetry::trace::SamplingDecision::RecordAndSample,
-                attributes: vec![],
-                trace_state: opentelemetry::trace::TraceState::default(),
-            }
-        } else {
-            opentelemetry::trace::SamplingResult {
-                decision: opentelemetry::trace::SamplingDecision::Drop,
-                attributes: vec![],
-                trace_state: opentelemetry::trace::TraceState::default(),
-            }
-        }
-    }
-}
-
 pub(crate) fn init_otel_tracing(service_name: &String, otel_endpoint: &String) {
-    println!(
+    tracing::info!(
         "Registering jaeger subscriber for {} at endpoint {}",
-        service_name, otel_endpoint
+        service_name,
+        otel_endpoint
     );
     let resource = opentelemetry_sdk::Resource::new(vec![opentelemetry::KeyValue::new(
         "service.name",
         service_name.clone(),
     )]);
 
     // Prepare tracer.
-    let client = reqwest::Client::new();
-    let span_exporter = opentelemetry_otlp::SpanExporter::builder()
-        .with_http()
-        .with_http_client(client)
+    let tracing_span_exporter = opentelemetry_otlp::SpanExporter::builder()
+        .with_tonic()
         .with_endpoint(otel_endpoint)
         .build()
-        .expect("could not build span exporter");
-    let trace_config = opentelemetry_sdk::trace::Config::default()
-        .with_sampler(ChromaShouldSample)
-        .with_resource(resource.clone());
+        .expect("could not build span exporter for tracing");
+    let trace_config = opentelemetry_sdk::trace::Config::default().with_resource(resource.clone());
     let tracer_provider = opentelemetry_sdk::trace::TracerProvider::builder()
-        .with_batch_exporter(span_exporter, opentelemetry_sdk::runtime::Tokio)
+        .with_batch_exporter(tracing_span_exporter, opentelemetry_sdk::runtime::Tokio)
         .with_config(trace_config)
         .build();
     let tracer = tracer_provider.tracer(service_name.clone());
-    // TODO(MrCroxx): Should we the tracer provider as global?
-    // global::set_tracer_provider(tracer_provider);
 
     // Prepare meter.
-    let mut headers = reqwest::header::HeaderMap::new();
-    headers.insert(
-        "x-honeycomb-dataset",
-        reqwest::header::HeaderValue::from_static("chroma-load"),
-    );
-    if let Ok(api_key) = std::env::var("OTEL_EXPORTER_OTLP_HEADERS") {
-        if let Some((header, value)) = api_key.split_once("=") {
-            if header == "x-honeycomb-team" {
-                headers.insert(
-                    reqwest::header::HeaderName::from_static("x-honeycomb-team"),
-                    reqwest::header::HeaderValue::from_bytes(value.trim().as_bytes()).unwrap(),
-                );
-            }
-        }
-    }
-    let client = reqwest::ClientBuilder::new()
-        .default_headers(headers)
-        .build()
-        .unwrap();
     let metric_exporter = opentelemetry_otlp::MetricExporter::builder()
-        .with_http()
-        .with_http_client(client)
-        .with_endpoint(otel_endpoint)
+        .with_tonic()
+        .with_endpoint(
+            std::env::var("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT").unwrap_or(otel_endpoint.clone()),
+        )
         .build()
         .expect("could not build metric exporter");