Add event processor functionality (#2420)

caaavik-msft · web-flow · commit ad937654174e · 2023-09-21T14:52:40.000+02:00
diff --git a/src/BenchmarkDotNet/Configs/ConfigExtensions.cs b/src/BenchmarkDotNet/Configs/ConfigExtensions.cs
@@ -7,6 +7,7 @@
 using BenchmarkDotNet.Analysers;
 using BenchmarkDotNet.Columns;
 using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.EventProcessors;
 using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Filters;
 using BenchmarkDotNet.Jobs;
@@ -110,6 +111,7 @@ public static class ConfigExtensions
         [Obsolete("This method will soon be removed, please start using .AddLogicalGroupRules() instead.")]
         [EditorBrowsable(EditorBrowsableState.Never)] public static IConfig With(this IConfig config, params BenchmarkLogicalGroupRule[] rules) => config.AddLogicalGroupRules(rules);
         [PublicAPI] public static ManualConfig AddLogicalGroupRules(this IConfig config, params BenchmarkLogicalGroupRule[] rules) => config.With(c => c.AddLogicalGroupRules(rules));
+        [PublicAPI] public static ManualConfig AddEventProcessor(this IConfig config, EventProcessor eventProcessor) => config.With(c => c.AddEventProcessor(eventProcessor));
 
         [PublicAPI] public static ManualConfig HideColumns(this IConfig config, params string[] columnNames) => config.With(c => c.HideColumns(columnNames));
         [PublicAPI] public static ManualConfig HideColumns(this IConfig config, params IColumn[] columns) => config.With(c => c.HideColumns(columns));
diff --git a/src/BenchmarkDotNet/Configs/DebugConfig.cs b/src/BenchmarkDotNet/Configs/DebugConfig.cs
@@ -4,6 +4,7 @@
 using BenchmarkDotNet.Analysers;
 using BenchmarkDotNet.Columns;
 using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.EventProcessors;
 using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Filters;
 using BenchmarkDotNet.Jobs;
@@ -61,6 +62,7 @@ public abstract class DebugConfig : IConfig
         public IEnumerable<IDiagnoser> GetDiagnosers() => Array.Empty<IDiagnoser>();
         public IEnumerable<IAnalyser> GetAnalysers() => Array.Empty<IAnalyser>();
         public IEnumerable<HardwareCounter> GetHardwareCounters() => Array.Empty<HardwareCounter>();
+        public IEnumerable<EventProcessor> GetEventProcessors() => Array.Empty<EventProcessor>();
         public IEnumerable<IFilter> GetFilters() => Array.Empty<IFilter>();
         public IEnumerable<IColumnHidingRule> GetColumnHidingRules() => Array.Empty<IColumnHidingRule>();
 
diff --git a/src/BenchmarkDotNet/Configs/DefaultConfig.cs b/src/BenchmarkDotNet/Configs/DefaultConfig.cs
@@ -5,6 +5,7 @@
 using BenchmarkDotNet.Analysers;
 using BenchmarkDotNet.Columns;
 using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.EventProcessors;
 using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Exporters.Csv;
 using BenchmarkDotNet.Filters;
@@ -108,6 +109,8 @@ public string ArtifactsPath
 
         public IEnumerable<IFilter> GetFilters() => Array.Empty<IFilter>();
 
+        public IEnumerable<EventProcessor> GetEventProcessors() => Array.Empty<EventProcessor>();
+
         public IEnumerable<IColumnHidingRule> GetColumnHidingRules() => Array.Empty<IColumnHidingRule>();
     }
 }
diff --git a/src/BenchmarkDotNet/Configs/IConfig.cs b/src/BenchmarkDotNet/Configs/IConfig.cs
@@ -4,6 +4,7 @@
 using BenchmarkDotNet.Analysers;
 using BenchmarkDotNet.Columns;
 using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.EventProcessors;
 using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Filters;
 using BenchmarkDotNet.Jobs;
@@ -27,6 +28,7 @@ public interface IConfig
         IEnumerable<HardwareCounter> GetHardwareCounters();
         IEnumerable<IFilter> GetFilters();
         IEnumerable<BenchmarkLogicalGroupRule> GetLogicalGroupRules();
+        IEnumerable<EventProcessor> GetEventProcessors();
         IEnumerable<IColumnHidingRule> GetColumnHidingRules();
 
         IOrderer? Orderer { get; }
diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs
@@ -6,6 +6,7 @@
 using BenchmarkDotNet.Analysers;
 using BenchmarkDotNet.Columns;
 using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.EventProcessors;
 using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Filters;
 using BenchmarkDotNet.Jobs;
@@ -31,6 +32,7 @@ public sealed class ImmutableConfig : IConfig
         private readonly ImmutableHashSet<HardwareCounter> hardwareCounters;
         private readonly ImmutableHashSet<IFilter> filters;
         private readonly ImmutableArray<BenchmarkLogicalGroupRule> rules;
+        private readonly ImmutableHashSet<EventProcessor> eventProcessors;
         private readonly ImmutableArray<IColumnHidingRule> columnHidingRules;
 
         internal ImmutableConfig(
@@ -45,6 +47,7 @@ internal ImmutableConfig(
             ImmutableArray<BenchmarkLogicalGroupRule> uniqueRules,
             ImmutableArray<IColumnHidingRule> uniqueColumnHidingRules,
             ImmutableHashSet<Job> uniqueRunnableJobs,
+            ImmutableHashSet<EventProcessor> uniqueEventProcessors,
             ConfigUnionRule unionRule,
             string artifactsPath,
             CultureInfo cultureInfo,
@@ -66,6 +69,7 @@ internal ImmutableConfig(
             rules = uniqueRules;
             columnHidingRules = uniqueColumnHidingRules;
             jobs = uniqueRunnableJobs;
+            eventProcessors = uniqueEventProcessors;
             UnionRule = unionRule;
             ArtifactsPath = artifactsPath;
             CultureInfo = cultureInfo;
@@ -96,6 +100,7 @@ internal ImmutableConfig(
         public IEnumerable<HardwareCounter> GetHardwareCounters() => hardwareCounters;
         public IEnumerable<IFilter> GetFilters() => filters;
         public IEnumerable<BenchmarkLogicalGroupRule> GetLogicalGroupRules() => rules;
+        public IEnumerable<EventProcessor> GetEventProcessors() => eventProcessors;
         public IEnumerable<IColumnHidingRule> GetColumnHidingRules() => columnHidingRules;
 
         public ILogger GetCompositeLogger() => new CompositeLogger(loggers);
diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs
@@ -53,6 +53,7 @@ public static ImmutableConfig Create(IConfig source)
             var uniqueHidingRules = source.GetColumnHidingRules().ToImmutableArray();
 
             var uniqueRunnableJobs = GetRunnableJobs(source.GetJobs()).ToImmutableHashSet();
+            var uniqueEventProcessors = source.GetEventProcessors().ToImmutableHashSet();
 
             return new ImmutableConfig(
                 uniqueColumnProviders,
@@ -66,6 +67,7 @@ public static ImmutableConfig Create(IConfig source)
                 uniqueRules,
                 uniqueHidingRules,
                 uniqueRunnableJobs,
+                uniqueEventProcessors,
                 source.UnionRule,
                 source.ArtifactsPath ?? DefaultConfig.Instance.ArtifactsPath,
                 source.CultureInfo,
diff --git a/src/BenchmarkDotNet/Configs/ManualConfig.cs b/src/BenchmarkDotNet/Configs/ManualConfig.cs
@@ -6,6 +6,7 @@
 using BenchmarkDotNet.Analysers;
 using BenchmarkDotNet.Columns;
 using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.EventProcessors;
 using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Extensions;
 using BenchmarkDotNet.Filters;
@@ -33,6 +34,7 @@ public class ManualConfig : IConfig
         private readonly HashSet<HardwareCounter> hardwareCounters = new HashSet<HardwareCounter>();
         private readonly List<IFilter> filters = new List<IFilter>();
         private readonly List<BenchmarkLogicalGroupRule> logicalGroupRules = new List<BenchmarkLogicalGroupRule>();
+        private readonly List<EventProcessor> eventProcessors = new List<EventProcessor>();
         private readonly List<IColumnHidingRule> columnHidingRules = new List<IColumnHidingRule>();
 
         public IEnumerable<IColumnProvider> GetColumnProviders() => columnProviders;
@@ -45,6 +47,7 @@ public class ManualConfig : IConfig
         public IEnumerable<HardwareCounter> GetHardwareCounters() => hardwareCounters;
         public IEnumerable<IFilter> GetFilters() => filters;
         public IEnumerable<BenchmarkLogicalGroupRule> GetLogicalGroupRules() => logicalGroupRules;
+        public IEnumerable<EventProcessor> GetEventProcessors() => eventProcessors;
         public IEnumerable<IColumnHidingRule> GetColumnHidingRules() => columnHidingRules;
 
         [PublicAPI] public ConfigOptions Options { get; set; }
@@ -221,6 +224,12 @@ public ManualConfig AddLogicalGroupRules(params BenchmarkLogicalGroupRule[] rule
             return this;
         }
 
+        public ManualConfig AddEventProcessor(EventProcessor eventProcessor)
+        {
+            this.eventProcessors.Add(eventProcessor);
+            return this;
+        }
+
         [PublicAPI]
         public ManualConfig HideColumns(params string[] columnNames)
         {
diff --git a/src/BenchmarkDotNet/EventProcessors/CompositeEventProcessor.cs b/src/BenchmarkDotNet/EventProcessors/CompositeEventProcessor.cs
@@ -0,0 +1,97 @@
+﻿using System;
+using System.Collections.Generic;
+using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Toolchains.Results;
+using BenchmarkDotNet.Validators;
+
+namespace BenchmarkDotNet.EventProcessors
+{
+    internal sealed class CompositeEventProcessor : EventProcessor
+    {
+        private readonly HashSet<EventProcessor> eventProcessors;
+
+        public CompositeEventProcessor(BenchmarkRunInfo[] benchmarkRunInfos)
+        {
+            var eventProcessors = new HashSet<EventProcessor>();
+
+            foreach (var info in benchmarkRunInfos)
+                eventProcessors.AddRange(info.Config.GetEventProcessors());
+
+            this.eventProcessors = eventProcessors;
+        }
+
+        public override void OnStartValidationStage()
+        {
+            foreach (var eventProcessor in eventProcessors)
+                eventProcessor.OnStartValidationStage();
+        }
+
+        public override void OnValidationError(ValidationError validationError)
+        {
+            foreach (var eventProcessor in eventProcessors)
+                eventProcessor.OnValidationError(validationError);
+        }
+
+        public override void OnEndValidationStage()
+        {
+            foreach (var eventProcessor in eventProcessors)
+                eventProcessor.OnEndValidationStage();
+        }
+
+        public override void OnStartBuildStage(IReadOnlyList<BuildPartition> partitions)
+        {
+            foreach (var eventProcessor in eventProcessors)
+                eventProcessor.OnStartBuildStage(partitions);
+        }
+
+        public override void OnBuildComplete(BuildPartition buildPartition, BuildResult buildResult)
+        {
+            foreach (var eventProcessor in eventProcessors)
+                eventProcessor.OnBuildComplete(buildPartition, buildResult);
+        }
+
+        public override void OnEndBuildStage()
+        {
+            foreach (var eventProcessor in eventProcessors)
+                eventProcessor.OnEndBuildStage();
+        }
+
+        public override void OnStartRunStage()
+        {
+            foreach (var eventProcessor in eventProcessors)
+                eventProcessor.OnStartRunStage();
+        }
+
+        public override void OnEndRunStage()
+        {
+            foreach (var eventProcessor in eventProcessors)
+                eventProcessor.OnEndRunStage();
+        }
+
+        public override void OnStartRunBenchmarksInType(Type type, IReadOnlyList<BenchmarkCase> benchmarks)
+        {
+            foreach (var eventProcessor in eventProcessors)
+                eventProcessor.OnStartRunBenchmarksInType(type, benchmarks);
+        }
+
+        public override void OnEndRunBenchmarksInType(Type type, Summary summary)
+        {
+            foreach (var eventProcessor in eventProcessors)
+                eventProcessor.OnEndRunBenchmarksInType(type, summary);
+        }
+
+        public override void OnEndRunBenchmark(BenchmarkCase benchmarkCase, BenchmarkReport report)
+        {
+            foreach (var eventProcessor in eventProcessors)
+                eventProcessor.OnEndRunBenchmark(benchmarkCase, report);
+        }
+
+        public override void OnStartRunBenchmark(BenchmarkCase benchmarkCase)
+        {
+            foreach (var eventProcessor in eventProcessors)
+                eventProcessor.OnStartRunBenchmark(benchmarkCase);
+        }
+    }
+}
diff --git a/src/BenchmarkDotNet/EventProcessors/EventProcessor.cs b/src/BenchmarkDotNet/EventProcessors/EventProcessor.cs
@@ -0,0 +1,25 @@
+﻿using System;
+using System.Collections.Generic;
+using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Toolchains.Results;
+using BenchmarkDotNet.Validators;
+
+namespace BenchmarkDotNet.EventProcessors
+{
+    public abstract class EventProcessor
+    {
+        public virtual void OnStartValidationStage() { }
+        public virtual void OnValidationError(ValidationError validationError) { }
+        public virtual void OnEndValidationStage() { }
+        public virtual void OnStartBuildStage(IReadOnlyList<BuildPartition> partitions) { }
+        public virtual void OnBuildComplete(BuildPartition partition, BuildResult buildResult) { }
+        public virtual void OnEndBuildStage() { }
+        public virtual void OnStartRunStage() { }
+        public virtual void OnStartRunBenchmarksInType(Type type, IReadOnlyList<BenchmarkCase> benchmarks) { }
+        public virtual void OnEndRunBenchmarksInType(Type type, Summary summary) { }
+        public virtual void OnStartRunBenchmark(BenchmarkCase benchmarkCase) { }
+        public virtual void OnEndRunBenchmark(BenchmarkCase benchmarkCase, BenchmarkReport report) { }
+        public virtual void OnEndRunStage() { }
+    }
+}
diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs
diff --git a/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs b/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs

Original file line number	Diff line number	Diff line change
`@@ -5,6 +5,7 @@`
`5`	`5`	`using BenchmarkDotNet.Analysers;`
`6`	`6`	`using BenchmarkDotNet.Columns;`
`7`	`7`	`using BenchmarkDotNet.Diagnosers;`
	`8`	`+using BenchmarkDotNet.EventProcessors;`
`8`	`9`	`using BenchmarkDotNet.Exporters;`
`9`	`10`	`using BenchmarkDotNet.Exporters.Csv;`
`10`	`11`	`using BenchmarkDotNet.Filters;`
`@@ -108,6 +109,8 @@ public string ArtifactsPath`
`108`	`109`
`109`	`110`	`public IEnumerable<IFilter> GetFilters() => Array.Empty<IFilter>();`
`110`	`111`
	`112`	`+ public IEnumerable<EventProcessor> GetEventProcessors() => Array.Empty<EventProcessor>();`
	`113`	`+`
`111`	`114`	`public IEnumerable<IColumnHidingRule> GetColumnHidingRules() => Array.Empty<IColumnHidingRule>();`
`112`	`115`	`}`
`113`	`116`	`}`