Skip to content

Commit b2a0f07

Browse files
openshift-merge-bot[bot]openshift-merge-bot[bot]
authored
Merge pull request #29932 from dgoodwin/high-cpu-e2e-analyzer
TRT-2172: Generate autodl artifact to study e2e tests correlated with high CPU
2 parents d9e17ff + f547212 commit b2a0f07

File tree

8 files changed

+848
-32
lines changed

8 files changed

+848
-32
lines changed

‎pkg/defaultmonitortests/types.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ import (
4747
"github.com/openshift/origin/pkg/monitortests/testframework/disruptionexternalservicemonitoring"
4848
"github.com/openshift/origin/pkg/monitortests/testframework/disruptionserializer"
4949
"github.com/openshift/origin/pkg/monitortests/testframework/e2etestanalyzer"
50+
"github.com/openshift/origin/pkg/monitortests/testframework/highcputestanalyzer"
5051

5152
"github.com/openshift/origin/pkg/monitortests/testframework/intervalserializer"
5253
"github.com/openshift/origin/pkg/monitortests/testframework/knownimagechecker"
@@ -202,6 +203,7 @@ func newUniversalMonitorTests(info monitortestframework.MonitorTestInitializatio
202203

203204
monitorTestRegistry.AddMonitorTestOrDie("azure-metrics-collector", "Test Framework", azuremetrics.NewAzureMetricsCollector())
204205
monitorTestRegistry.AddMonitorTestOrDie("watch-namespaces", "Test Framework", watchnamespaces.NewNamespaceWatcher())
206+
monitorTestRegistry.AddMonitorTestOrDie("high-cpu-test-analyzer", "Test Framework", highcputestanalyzer.NewHighCPUTestAnalyzer())
205207

206208
return monitorTestRegistry
207209
}

‎pkg/monitortestlibrary/utility/utility.go

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import (
55
"os"
66
"regexp"
77
"time"
8+
9+
"github.com/openshift/origin/pkg/monitor/monitorapi"
810
)
911

1012
// SystemdJournalLogTime returns Now if there is trouble reading the time. This will stack the event intervals without
@@ -42,3 +44,33 @@ func SystemdJournalLogTime(logLine string, year int) time.Time {
4244

4345
return ret
4446
}
47+
48+
// FindOverlap finds any intervals that overlap with the given interval.
49+
func FindOverlap(intervals monitorapi.Intervals, overlapsWith monitorapi.Interval) monitorapi.Intervals {
50+
overlappingIntervals := monitorapi.Intervals{}
51+
for i := range intervals {
52+
interval := intervals[i]
53+
if IntervalsOverlap(interval, overlapsWith) {
54+
overlappingIntervals = append(overlappingIntervals, interval)
55+
}
56+
}
57+
58+
return overlappingIntervals
59+
}
60+
61+
// IntervalsOverlap checks if two intervals overlap in time
62+
func IntervalsOverlap(interval1, interval2 monitorapi.Interval) bool {
63+
// If either interval has a zero end time, treat it as ongoing to the end of time
64+
end1 := interval1.To
65+
if end1.IsZero() {
66+
end1 = time.Date(9999, 12, 31, 23, 59, 59, 999999999, time.UTC)
67+
}
68+
69+
end2 := interval2.To
70+
if end2.IsZero() {
71+
end2 = time.Date(9999, 12, 31, 23, 59, 59, 999999999, time.UTC)
72+
}
73+
74+
// Check for overlap
75+
return (interval1.From.Before(end2)) && (interval2.From.Before(end1))
76+
}

‎pkg/monitortestlibrary/utility/utility_test.go

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
package utility
2+
3+
import (
4+
"testing"
5+
"time"
6+
7+
"github.com/openshift/origin/pkg/monitor/monitorapi"
8+
"github.com/stretchr/testify/assert"
9+
)
10+
11+
func TestOverlaps(t *testing.T) {
12+
now := time.Now()
13+
14+
testCases := []struct {
15+
name string
16+
interval1 monitorapi.Interval
17+
interval2 monitorapi.Interval
18+
expected bool
19+
}{
20+
{
21+
name: "intervals overlap",
22+
interval1: monitorapi.Interval{
23+
From: now,
24+
To: now.Add(10 * time.Minute),
25+
},
26+
interval2: monitorapi.Interval{
27+
From: now.Add(5 * time.Minute),
28+
To: now.Add(15 * time.Minute),
29+
},
30+
expected: true,
31+
},
32+
{
33+
name: "interval1 contains interval2",
34+
interval1: monitorapi.Interval{
35+
From: now,
36+
To: now.Add(20 * time.Minute),
37+
},
38+
interval2: monitorapi.Interval{
39+
From: now.Add(5 * time.Minute),
40+
To: now.Add(15 * time.Minute),
41+
},
42+
expected: true,
43+
},
44+
{
45+
name: "interval2 contains interval1",
46+
interval1: monitorapi.Interval{
47+
From: now.Add(5 * time.Minute),
48+
To: now.Add(15 * time.Minute),
49+
},
50+
interval2: monitorapi.Interval{
51+
From: now,
52+
To: now.Add(20 * time.Minute),
53+
},
54+
expected: true,
55+
},
56+
{
57+
name: "intervals touch at start",
58+
interval1: monitorapi.Interval{
59+
From: now,
60+
To: now.Add(10 * time.Minute),
61+
},
62+
interval2: monitorapi.Interval{
63+
From: now.Add(10 * time.Minute),
64+
To: now.Add(20 * time.Minute),
65+
},
66+
expected: false,
67+
},
68+
{
69+
name: "intervals touch at end",
70+
interval1: monitorapi.Interval{
71+
From: now.Add(10 * time.Minute),
72+
To: now.Add(20 * time.Minute),
73+
},
74+
interval2: monitorapi.Interval{
75+
From: now,
76+
To: now.Add(10 * time.Minute),
77+
},
78+
expected: false,
79+
},
80+
{
81+
name: "intervals don't overlap",
82+
interval1: monitorapi.Interval{
83+
From: now,
84+
To: now.Add(10 * time.Minute),
85+
},
86+
interval2: monitorapi.Interval{
87+
From: now.Add(11 * time.Minute),
88+
To: now.Add(20 * time.Minute),
89+
},
90+
expected: false,
91+
},
92+
{
93+
name: "interval1 has zero end time",
94+
interval1: monitorapi.Interval{
95+
From: now,
96+
To: time.Time{},
97+
},
98+
interval2: monitorapi.Interval{
99+
From: now.Add(10 * time.Minute),
100+
To: now.Add(20 * time.Minute),
101+
},
102+
expected: true,
103+
},
104+
{
105+
name: "interval2 has zero end time",
106+
interval1: monitorapi.Interval{
107+
From: now,
108+
To: now.Add(10 * time.Minute),
109+
},
110+
interval2: monitorapi.Interval{
111+
From: now.Add(5 * time.Minute),
112+
To: time.Time{},
113+
},
114+
expected: true,
115+
},
116+
{
117+
name: "both intervals have zero end time",
118+
interval1: monitorapi.Interval{
119+
From: now,
120+
To: time.Time{},
121+
},
122+
interval2: monitorapi.Interval{
123+
From: now.Add(10 * time.Minute),
124+
To: time.Time{},
125+
},
126+
expected: true,
127+
},
128+
}
129+
130+
for _, tc := range testCases {
131+
t.Run(tc.name, func(t *testing.T) {
132+
result := IntervalsOverlap(tc.interval1, tc.interval2)
133+
assert.Equal(t, tc.expected, result)
134+
})
135+
}
136+
}

‎pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"strings"
77
"time"
88

9+
"github.com/openshift/origin/pkg/monitortestlibrary/utility"
910
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1011
"k8s.io/client-go/kubernetes"
1112

@@ -519,7 +520,7 @@ func testOperatorStateTransitions(events monitorapi.Intervals, conditionTypes []
519520
// if there was any switch, it was wrong/unexpected at some point
520521
failure := fmt.Sprintf("%v", eventInterval)
521522

522-
overlappingE2EIntervals := operatorstateanalyzer.FindOverlap(e2eEventIntervals, eventInterval.From, eventInterval.From)
523+
overlappingE2EIntervals := utility.FindOverlap(e2eEventIntervals, eventInterval)
523524
concurrentE2E := []string{}
524525
for _, overlap := range overlappingE2EIntervals {
525526
if overlap.Level == monitorapi.Info {
@@ -533,7 +534,7 @@ func testOperatorStateTransitions(events monitorapi.Intervals, conditionTypes []
533534
}
534535

535536
if len(concurrentE2E) > 0 {
536-
failure = fmt.Sprintf("%s\n%d tests failed during this blip (%v to %v): %v", failure, len(concurrentE2E), eventInterval.From, eventInterval.From, strings.Join(concurrentE2E, "\n"))
537+
failure = fmt.Sprintf("%s\n%d tests failed during this blip (%v to %v): %v", failure, len(concurrentE2E), eventInterval.From, eventInterval.To, strings.Join(concurrentE2E, "\n"))
537538
}
538539
exception, err := except(operatorName, condition, eventInterval, clientConfig)
539540
if err != nil || exception == "" {

‎pkg/monitortests/clusterversionoperator/operatorstateanalyzer/e2etest.go

Lines changed: 0 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
package operatorstateanalyzer
22

33
import (
4-
"time"
5-
64
"github.com/openshift/origin/pkg/monitor/monitorapi"
75
)
86

@@ -21,29 +19,3 @@ func E2ETestEventIntervals(events monitorapi.Intervals) monitorapi.Intervals {
2119
}
2220
return e2eEventIntervals
2321
}
24-
25-
// FindOverlap finds intervals that overlap with the time between start and end.
26-
func FindOverlap(intervals monitorapi.Intervals, start, end time.Time) monitorapi.Intervals {
27-
overlappingIntervals := monitorapi.Intervals{}
28-
for i := range intervals {
29-
interval := intervals[i]
30-
switch {
31-
case interval.From.After(start) && interval.From.Before(end):
32-
// if the interval started during the window, we overlapped
33-
overlappingIntervals = append(overlappingIntervals, interval)
34-
case interval.To.After(start) && interval.To.Before(end):
35-
// if the interval ended during the window, we overlapped
36-
overlappingIntervals = append(overlappingIntervals, interval)
37-
38-
case interval.From.Before(start) && interval.To.After(end):
39-
// if the interval started before the window and ended after the window, we overlapped
40-
overlappingIntervals = append(overlappingIntervals, interval)
41-
42-
default:
43-
// the other two cases are starting and ending before the window (no overlap) and
44-
// starting and ending after the window (no overlap)
45-
}
46-
}
47-
48-
return overlappingIntervals
49-
}

0 commit comments

Comments
 (0)