Merge branch 'main' into stanley.liu/otel-e2e-pipelines

Author: liustanley

cmd/otel-agent/config/agent_config.go

@@ -8,6 +8,7 @@ package config
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strings"
 
@@ -59,8 +60,14 @@ var logLevelReverseMap = func(src map[string]logLevel) map[logLevel]string {
 	return reverse
 }(logLevelMap)
 
+// ErrNoDDExporter indicates there is no Datadog exporter in the configs
+var ErrNoDDExporter = fmt.Errorf("no datadog exporter found")
+
 // NewConfigComponent creates a new config component from the given URIs
 func NewConfigComponent(ctx context.Context, ddCfg string, uris []string) (config.Component, error) {
+	if len(uris) == 0 {
+		return nil, errors.New("no URIs provided for configs")
+	}
 	// Load the configuration from the fileName
 	rs := confmap.ResolverSettings{
 		URIs: uris,
@@ -220,7 +227,7 @@ func getDDExporterConfig(cfg *confmap.Conf) (*datadogexporter.Config, error) {
 		}
 	}
 	if len(configs) == 0 {
-		return nil, fmt.Errorf("no datadog exporter found")
+		return nil, ErrNoDDExporter
 	}
 	// Check if we have multiple datadog exporters
 	// We only support one exporter for now

cmd/otel-agent/config/agent_config_test.go

@@ -29,6 +29,11 @@ func (suite *ConfigTestSuite) SetupTest() {
 	pkgconfigsetup.SetDatadog(datadog)
 }
 
+func TestNoURIsProvided(t *testing.T) {
+	_, err := NewConfigComponent(context.Background(), "", []string{})
+	assert.Error(t, err, "no URIs provided for configs")
+}
+
 func (suite *ConfigTestSuite) TestAgentConfig() {
 	t := suite.T()
 	fileName := "testdata/config.yaml"

cmd/otel-agent/subcommands/run/command.go

@@ -92,7 +92,24 @@ func (o *orchestratorinterfaceimpl) Reset() {
 }
 
 func runOTelAgentCommand(ctx context.Context, params *subcommands.GlobalParams, opts ...fx.Option) error {
-	err := fxutil.Run(
+	acfg, err := agentConfig.NewConfigComponent(context.Background(), params.CoreConfPath, params.ConfPaths)
+	if err != nil && err != agentConfig.ErrNoDDExporter {
+		return err
+	}
+	if err == agentConfig.ErrNoDDExporter {
+		return fxutil.Run(
+			fx.Provide(func() []string {
+				return append(params.ConfPaths, params.Sets...)
+			}),
+			collectorcontribFx.Module(),
+			collectorfx.ModuleNoAgent(),
+			fx.Options(opts...),
+			fx.Invoke(func(_ collectordef.Component) {
+			}),
+		)
+	}
+
+	return fxutil.Run(
 		ForwarderBundle(),
 		logtracefx.Module(),
 		inventoryagentimpl.Module(),
@@ -108,12 +125,8 @@ func runOTelAgentCommand(ctx context.Context, params *subcommands.GlobalParams,
 			return cp
 		}),
 		fx.Provide(func() (coreconfig.Component, error) {
-			c, err := agentConfig.NewConfigComponent(context.Background(), params.CoreConfPath, params.ConfPaths)
-			if err != nil {
-				return nil, err
-			}
-			pkgconfigenv.DetectFeatures(c)
-			return c, nil
+			pkgconfigenv.DetectFeatures(acfg)
+			return acfg, nil
 		}),
 		workloadmetafx.Module(workloadmeta.Params{
 			AgentType:  workloadmeta.NodeAgent,
@@ -197,10 +210,6 @@ func runOTelAgentCommand(ctx context.Context, params *subcommands.GlobalParams,
 		}),
 		traceagentfx.Module(),
 	)
-	if err != nil {
-		return err
-	}
-	return nil
 }
 
 // ForwarderBundle returns the fx.Option for the forwarder bundle.

cmd/otel-agent/subcommands/run/command_test.go

@@ -15,10 +15,22 @@ import (
 	"github.com/DataDog/datadog-agent/pkg/util/fxutil"
 )
 
-func TestFxRun(t *testing.T) {
+func TestFxRun_WithDatadogExporter(t *testing.T) {
 	fxutil.TestRun(t, func() error {
 		ctx := context.Background()
-		cliParams := &subcommands.GlobalParams{}
-		return runOTelAgentCommand(ctx, cliParams)
+		params := &subcommands.GlobalParams{
+			ConfPaths: []string{"test_config.yaml"},
+		}
+		return runOTelAgentCommand(ctx, params)
+	})
+}
+
+func TestFxRun_NoDatadogExporter(t *testing.T) {
+	fxutil.TestRun(t, func() error {
+		ctx := context.Background()
+		params := &subcommands.GlobalParams{
+			ConfPaths: []string{"test_config_no_dd.yaml"},
+		}
+		return runOTelAgentCommand(ctx, params)
 	})
 }

cmd/otel-agent/subcommands/run/test_config.yaml

@@ -0,0 +1,21 @@
+receivers:
+  otlp:
+    protocols:
+      http:
+        endpoint: "localhost:4318"
+      grpc:
+        endpoint: "localhost:4317"
+
+exporters:
+  datadog:
+    api:
+      key: "abc"
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [datadog]
+  telemetry:
+    metrics:
+      address: 127.0.0.1:8888

cmd/otel-agent/subcommands/run/test_config_no_dd.yaml

@@ -0,0 +1,19 @@
+receivers:
+  otlp:
+    protocols:
+      http:
+        endpoint: "localhost:4318"
+      grpc:
+        endpoint: "localhost:4317"
+
+exporters:
+  debug:
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [debug]
+  telemetry:
+    metrics:
+      address: 127.0.0.1:8888

comp/otelcol/collector/fx/fx.go

@@ -25,3 +25,13 @@ func Module() fxutil.Module {
 		fxutil.ProvideOptional[collector.Component](),
 	)
 }
+
+// ModuleNoAgent for OTel Agent with no Agent functionalities
+func ModuleNoAgent() fxutil.Module {
+	return fxutil.Component(
+		fxutil.ProvideComponentConstructor(
+			collectorimpl.NewComponentNoAgent,
+		),
+		fxutil.ProvideOptional[collector.Component](),
+	)
+}

comp/otelcol/collector/impl/collector.go

@@ -10,6 +10,7 @@ package collectorimpl
 
 import (
 	"context"
+	"log"
 	"os"
 	"path/filepath"
 
@@ -28,7 +29,7 @@ import (
 	"go.uber.org/zap/zapcore"
 
 	"github.com/DataDog/datadog-agent/comp/core/config"
-	log "github.com/DataDog/datadog-agent/comp/core/log/def"
+	corelog "github.com/DataDog/datadog-agent/comp/core/log/def"
 	"github.com/DataDog/datadog-agent/comp/core/tagger"
 	"github.com/DataDog/datadog-agent/comp/core/workloadmeta/collectors/util"
 	compdef "github.com/DataDog/datadog-agent/comp/def"
@@ -48,7 +49,7 @@ import (
 )
 
 type collectorImpl struct {
-	log log.Component
+	log corelog.Component
 	set otelcol.CollectorSettings
 	col *otelcol.Collector
 }
@@ -59,18 +60,29 @@ type Requires struct {
 	// and shutdown hooks.
 	Lc compdef.Lifecycle
 
-	// Log specifies the logging component.
-	Log                 log.Component
-	Provider            confmap.Converter
+	CollectorContrib collectorcontrib.Component
+	URIs             []string
+
+	// Below are dependencies required by Datadog exporter and other Agent functionalities
+	Log                 corelog.Component
+	Converter           confmap.Converter
 	Config              config.Component
-	CollectorContrib    collectorcontrib.Component
 	Serializer          serializer.MetricSerializer
 	TraceAgent          traceagent.Component
 	LogsAgent           optional.Option[logsagentpipeline.Component]
 	SourceProvider      serializerexporter.SourceProviderFunc
 	Tagger              tagger.Component
 	StatsdClientWrapper *metricsclient.StatsdClientWrapper
-	URIs                []string
+}
+
+// RequiresNoAgent declares the input types to the constructor with no dependencies on Agent components
+type RequiresNoAgent struct {
+	// Lc specifies the compdef lifecycle settings, used for appending startup
+	// and shutdown hooks.
+	Lc compdef.Lifecycle
+
+	CollectorContrib collectorcontrib.Component
+	URIs             []string
 }
 
 // Provides declares the output types from the constructor
@@ -88,18 +100,18 @@ func (c *converterFactory) Create(_ confmap.ConverterSettings) confmap.Converter
 	return c.converter
 }
 
-func newConfigProviderSettings(reqs Requires, enhanced bool) otelcol.ConfigProviderSettings {
+func newConfigProviderSettings(uris []string, converter confmap.Converter, enhanced bool) otelcol.ConfigProviderSettings {
 	converterFactories := []confmap.ConverterFactory{
 		expandconverter.NewFactory(),
 	}
 
 	if enhanced {
-		converterFactories = append(converterFactories, &converterFactory{converter: reqs.Provider})
+		converterFactories = append(converterFactories, &converterFactory{converter: converter})
 	}
 
 	return otelcol.ConfigProviderSettings{
 		ResolverSettings: confmap.ResolverSettings{
-			URIs: reqs.URIs,
+			URIs: uris,
 			ProviderFactories: []confmap.ProviderFactory{
 				fileprovider.NewFactory(),
 				envprovider.NewFactory(),
@@ -124,10 +136,16 @@ func addFactories(reqs Requires, factories otelcol.Factories) {
 	}
 	factories.Processors[infraattributesprocessor.Type] = infraattributesprocessor.NewFactory(reqs.Tagger, generateID)
 	factories.Connectors[component.MustNewType("datadog")] = datadogconnector.NewFactory()
-	factories.Extensions[ddextension.Type] = ddextension.NewFactory(&factories, newConfigProviderSettings(reqs, false))
+	factories.Extensions[ddextension.Type] = ddextension.NewFactory(&factories, newConfigProviderSettings(reqs.URIs, reqs.Converter, false))
 }
 
-// NewComponent returns a new instance of the collector component.
+var buildInfo = component.BuildInfo{
+	Version:     "v0.104.0",
+	Command:     filepath.Base(os.Args[0]),
+	Description: "Datadog Agent OpenTelemetry Collector",
+}
+
+// NewComponent returns a new instance of the collector component with full Agent functionalities.
 func NewComponent(reqs Requires) (Provides, error) {
 	factories, err := reqs.CollectorContrib.OTelComponentFactories()
 	if err != nil {
@@ -143,16 +161,12 @@ func NewComponent(reqs Requires) (Provides, error) {
 		}),
 	}
 	set := otelcol.CollectorSettings{
-		BuildInfo: component.BuildInfo{
-			Version:     "v0.104.0",
-			Command:     filepath.Base(os.Args[0]),
-			Description: "Datadog Agent OpenTelemetry Collector",
-		},
+		BuildInfo:      buildInfo,
 		LoggingOptions: options,
 		Factories: func() (otelcol.Factories, error) {
 			return factories, nil
 		},
-		ConfigProviderSettings: newConfigProviderSettings(reqs, converterEnabled),
+		ConfigProviderSettings: newConfigProviderSettings(reqs.URIs, reqs.Converter, converterEnabled),
 	}
 	col, err := otelcol.NewCollector(set)
 	if err != nil {
@@ -173,6 +187,39 @@ func NewComponent(reqs Requires) (Provides, error) {
 	}, nil
 }
 
+// NewComponentNoAgent returns a new instance of the collector component with no Agent functionalities.
+// It is used when there is no Datadog exporter in the OTel Agent config.
+func NewComponentNoAgent(reqs RequiresNoAgent) (Provides, error) {
+	factories, err := reqs.CollectorContrib.OTelComponentFactories()
+	if err != nil {
+		return Provides{}, err
+	}
+
+	set := otelcol.CollectorSettings{
+		BuildInfo: buildInfo,
+		Factories: func() (otelcol.Factories, error) {
+			return factories, nil
+		},
+		ConfigProviderSettings: newConfigProviderSettings(reqs.URIs, nil, false),
+	}
+	col, err := otelcol.NewCollector(set)
+	if err != nil {
+		return Provides{}, err
+	}
+	c := &collectorImpl{
+		set: set,
+		col: col,
+	}
+
+	reqs.Lc.Append(compdef.Hook{
+		OnStart: c.start,
+		OnStop:  c.stop,
+	})
+	return Provides{
+		Comp: c,
+	}, nil
+}
+
 func (c *collectorImpl) start(ctx context.Context) error {
 	// Dry run the collector pipeline to ensure it is configured correctly
 	err := c.col.DryRun(ctx)
@@ -181,7 +228,11 @@ func (c *collectorImpl) start(ctx context.Context) error {
 	}
 	go func() {
 		if err := c.col.Run(context.Background()); err != nil {
-			c.log.Errorf("Error running the collector pipeline: %v", err)
+			if c.log != nil {
+				c.log.Errorf("Error running the collector pipeline: %v", err)
+			} else {
+				log.Printf("Error running the collector pipeline: %v", err)
+			}
 		}
 	}()
 	return nil

tools/windows/DatadogAgentInstaller/AgentCustomActions/InstallStateCustomActions.cs

@@ -75,7 +75,9 @@ private void SetDDDriverRollback()
             var upgradeDetected = _session["WIX_UPGRADE_DETECTED"];
             if (!string.IsNullOrEmpty(upgradeDetected)) // This is an upgrade, conditionally set rollback flags.
             {
+                _session.Log($"WIX_UPGRADE_DETECTED: {_session["WIX_UPGRADE_DETECTED"]}");
                 var versionString = _nativeMethods.GetVersionString(upgradeDetected);
+                _session.Log($"versionString: {versionString}");
                 // Using Version class
                 // https://learn.microsoft.com/en-us/dotnet/api/system.version?view=net-8.0
                 var currentVersion = new Version(versionString);

tools/windows/DatadogAgentInstaller/CustomActions/Native/NativeMethods.cs

@@ -930,7 +930,12 @@ public string GetVersionString(string product)
             var len = 512;
             var builder = new System.Text.StringBuilder(len);
 
-            MsiGetProductInfo(product, "VersionString", builder, ref len);
+            var err = (ReturnCodes)MsiGetProductInfo(product, "VersionString", builder, ref len);
+            if (err != ReturnCodes.NO_ERROR)
+            {
+                throw new Exception($"Failed to get version string for product {product}",
+                    new Win32Exception((int)err));
+            }
 
             return builder.ToString();
         }