Skip to content

Commit 1efaf29

Browse files
flavio-fernandesflavio-fernandes
committed
Implement NetworkQoS controller
This commit implements Network QoS controller, based on the enhancement ovn-kubernetes/ovn-kubernetes#4366 Signed-off-by: Flavio Fernandes <[email protected]> Signed-off-by: Xiaobin Qu <[email protected]>
1 parent 11c7f0d commit 1efaf29

27 files changed

+3540
-8
lines changed

‎go-controller/pkg/clustermanager/status_manager/networkqos_manager.go

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
package status_manager
2+
3+
import (
4+
"context"
5+
"strings"
6+
7+
networkqosapi "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/networkqos/v1alpha1"
8+
networkqosapply "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/networkqos/v1alpha1/apis/applyconfiguration/networkqos/v1alpha1"
9+
networkqosclientset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/networkqos/v1alpha1/apis/clientset/versioned"
10+
networkqoslisters "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/networkqos/v1alpha1/apis/listers/networkqos/v1alpha1"
11+
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types"
12+
13+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
14+
)
15+
16+
type networkQoSManager struct {
17+
lister networkqoslisters.NetworkQoSLister
18+
client networkqosclientset.Interface
19+
}
20+
21+
func newNetworkQoSManager(lister networkqoslisters.NetworkQoSLister, client networkqosclientset.Interface) *networkQoSManager {
22+
return &networkQoSManager{
23+
lister: lister,
24+
client: client,
25+
}
26+
}
27+
28+
//lint:ignore U1000 generic interfaces throw false-positives https://github.com/dominikh/go-tools/issues/1440
29+
func (m *networkQoSManager) get(namespace, name string) (*networkqosapi.NetworkQoS, error) {
30+
return m.lister.NetworkQoSes(namespace).Get(name)
31+
}
32+
33+
//lint:ignore U1000 generic interfaces throw false-positives
34+
func (m *networkQoSManager) getMessages(networkQoS *networkqosapi.NetworkQoS) []string {
35+
var messages []string
36+
for _, condition := range networkQoS.Status.Conditions {
37+
messages = append(messages, condition.Message)
38+
}
39+
return messages
40+
}
41+
42+
//lint:ignore U1000 generic interfaces throw false-positives
43+
func (m *networkQoSManager) updateStatus(networkQoS *networkqosapi.NetworkQoS, applyOpts *metav1.ApplyOptions,
44+
applyEmptyOrFailed bool) error {
45+
if networkQoS == nil {
46+
return nil
47+
}
48+
newStatus := "NetworkQoS Destinations applied"
49+
for _, condition := range networkQoS.Status.Conditions {
50+
if strings.Contains(condition.Message, types.NetworkQoSErrorMsg) {
51+
newStatus = types.NetworkQoSErrorMsg
52+
break
53+
}
54+
}
55+
if applyEmptyOrFailed && newStatus != types.NetworkQoSErrorMsg {
56+
newStatus = ""
57+
}
58+
59+
if networkQoS.Status.Status == newStatus {
60+
// already set to the same value
61+
return nil
62+
}
63+
64+
applyStatus := networkqosapply.Status()
65+
if newStatus != "" {
66+
applyStatus.WithStatus(newStatus)
67+
}
68+
69+
applyObj := networkqosapply.NetworkQoS(networkQoS.Name, networkQoS.Namespace).
70+
WithStatus(applyStatus)
71+
72+
_, err := m.client.K8sV1alpha1().NetworkQoSes(networkQoS.Namespace).ApplyStatus(context.TODO(), applyObj, *applyOpts)
73+
return err
74+
}
75+
76+
//lint:ignore U1000 generic interfaces throw false-positives
77+
func (m *networkQoSManager) cleanupStatus(networkQoS *networkqosapi.NetworkQoS, applyOpts *metav1.ApplyOptions) error {
78+
applyObj := networkqosapply.NetworkQoS(networkQoS.Name, networkQoS.Namespace).
79+
WithStatus(networkqosapply.Status())
80+
81+
_, err := m.client.K8sV1alpha1().NetworkQoSes(networkQoS.Namespace).ApplyStatus(context.TODO(), applyObj, *applyOpts)
82+
return err
83+
}

‎go-controller/pkg/clustermanager/status_manager/status_manager.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
adminpolicybasedrouteapi "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1"
2121
egressfirewallapi "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1"
2222
egressqosapi "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1"
23+
networkqosapi "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/networkqos/v1alpha1"
2324
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory"
2425
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types"
2526
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"
@@ -203,6 +204,16 @@ func NewStatusManager(wf *factory.WatchFactory, ovnClient *util.OVNClusterManage
203204
)
204205
sm.typedManagers["egressqoses"] = egressQoSManager
205206
}
207+
if config.OVNKubernetesFeature.EnableNetworkQoS {
208+
networkQoSManager := newStatusManager[networkqosapi.NetworkQoS](
209+
"networkqoses_statusmanager",
210+
wf.NetworkQoSInformer().Informer(),
211+
wf.NetworkQoSInformer().Lister().List,
212+
newNetworkQoSManager(wf.NetworkQoSInformer().Lister(), ovnClient.NetworkQoSClient),
213+
sm.withZonesRLock,
214+
)
215+
sm.typedManagers["networkqoses"] = networkQoSManager
216+
}
206217
return sm
207218
}
208219

‎go-controller/pkg/clustermanager/status_manager/status_manager_test.go

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,14 @@ import (
2121
adminpolicybasedrouteapi "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1"
2222
egressfirewallapi "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1"
2323
egressqosapi "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1"
24+
networkqosapi "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/networkqos/v1alpha1"
2425
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory"
2526
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types"
2627
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"
2728

2829
. "github.com/onsi/ginkgo/v2"
2930
. "github.com/onsi/gomega"
31+
networkingv1 "k8s.io/api/networking/v1"
3032
)
3133

3234
func getNodeWithZone(nodeName, zoneName string) *corev1.Node {
@@ -203,6 +205,72 @@ func checkEmptyEQStatusConsistently(egressQoS *egressqosapi.EgressQoS, fakeClien
203205
}).Should(BeTrue(), "expected Status to be consistently empty")
204206
}
205207

208+
func newNetworkQoS(namespace string) *networkqosapi.NetworkQoS {
209+
return &networkqosapi.NetworkQoS{
210+
ObjectMeta: util.NewObjectMeta("default", namespace),
211+
Spec: networkqosapi.Spec{
212+
NetworkAttachmentRefs: []v1.ObjectReference{
213+
{
214+
Kind: "NetworkAttachmentDefinition",
215+
Namespace: "default",
216+
Name: "stream",
217+
},
218+
},
219+
Priority: 100,
220+
Egress: []networkqosapi.Rule{
221+
{
222+
DSCP: 60,
223+
Classifier: networkqosapi.Classifier{
224+
To: []networkqosapi.Destination{
225+
{
226+
IPBlock: &networkingv1.IPBlock{
227+
CIDR: "1.2.3.4/32",
228+
},
229+
},
230+
},
231+
},
232+
Bandwidth: networkqosapi.Bandwidth{
233+
Rate: 100,
234+
Burst: 1000,
235+
},
236+
},
237+
},
238+
},
239+
}
240+
}
241+
242+
func updateNetworkQoSStatus(networkQoS *networkqosapi.NetworkQoS, status *networkqosapi.Status,
243+
fakeClient *util.OVNClusterManagerClientset) {
244+
networkQoS.Status = *status
245+
_, err := fakeClient.NetworkQoSClient.K8sV1alpha1().NetworkQoSes(networkQoS.Namespace).
246+
Update(context.TODO(), networkQoS, metav1.UpdateOptions{})
247+
Expect(err).ToNot(HaveOccurred())
248+
}
249+
250+
func checkNQStatusEventually(networkQoS *networkqosapi.NetworkQoS, expectFailure bool, expectEmpty bool, fakeClient *util.OVNClusterManagerClientset) {
251+
Eventually(func() bool {
252+
eq, err := fakeClient.NetworkQoSClient.K8sV1alpha1().NetworkQoSes(networkQoS.Namespace).
253+
Get(context.TODO(), networkQoS.Name, metav1.GetOptions{})
254+
Expect(err).NotTo(HaveOccurred())
255+
if expectFailure {
256+
return strings.Contains(eq.Status.Status, types.NetworkQoSErrorMsg)
257+
} else if expectEmpty {
258+
return eq.Status.Status == ""
259+
} else {
260+
return strings.Contains(eq.Status.Status, "applied")
261+
}
262+
}).Should(BeTrue(), fmt.Sprintf("expected network QoS status with expectFailure=%v expectEmpty=%v", expectFailure, expectEmpty))
263+
}
264+
265+
func checkEmptyNQStatusConsistently(networkQoS *networkqosapi.NetworkQoS, fakeClient *util.OVNClusterManagerClientset) {
266+
Consistently(func() bool {
267+
ef, err := fakeClient.NetworkQoSClient.K8sV1alpha1().NetworkQoSes(networkQoS.Namespace).
268+
Get(context.TODO(), networkQoS.Name, metav1.GetOptions{})
269+
Expect(err).NotTo(HaveOccurred())
270+
return ef.Status.Status == ""
271+
}).Should(BeTrue(), "expected Status to be consistently empty")
272+
}
273+
206274
var _ = Describe("Cluster Manager Status Manager", func() {
207275
var (
208276
statusManager *StatusManager
@@ -505,4 +573,96 @@ var _ = Describe("Cluster Manager Status Manager", func() {
505573
return atomic.LoadUint32(&banpWerePatched)
506574
}).Should(Equal(uint32(2)))
507575
})
576+
577+
It("updates NetworkQoS status with 1 zone", func() {
578+
config.OVNKubernetesFeature.EnableNetworkQoS = true
579+
zones := sets.New[string]("zone1")
580+
namespace1 := util.NewNamespace(namespace1Name)
581+
networkQoS := newNetworkQoS(namespace1.Name)
582+
start(zones, namespace1, networkQoS)
583+
updateNetworkQoSStatus(networkQoS, &networkqosapi.Status{
584+
Conditions: []metav1.Condition{{
585+
Type: "Ready-In-Zone-zone1",
586+
Status: metav1.ConditionTrue,
587+
Reason: "SetupSucceeded",
588+
Message: "NetworkQoS Destinations applied",
589+
}},
590+
}, fakeClient)
591+
592+
checkNQStatusEventually(networkQoS, false, false, fakeClient)
593+
})
594+
595+
It("updates NetworkQoS status with 2 zones", func() {
596+
config.OVNKubernetesFeature.EnableNetworkQoS = true
597+
zones := sets.New[string]("zone1", "zone2")
598+
namespace1 := util.NewNamespace(namespace1Name)
599+
networkQoS := newNetworkQoS(namespace1.Name)
600+
start(zones, namespace1, networkQoS)
601+
602+
updateNetworkQoSStatus(networkQoS, &networkqosapi.Status{
603+
Conditions: []metav1.Condition{{
604+
Type: "Ready-In-Zone-zone1",
605+
Status: metav1.ConditionTrue,
606+
Reason: "SetupSucceeded",
607+
Message: "NetworkQoS Destinations applied",
608+
}},
609+
}, fakeClient)
610+
611+
checkEmptyNQStatusConsistently(networkQoS, fakeClient)
612+
613+
updateNetworkQoSStatus(networkQoS, &networkqosapi.Status{
614+
Conditions: []metav1.Condition{{
615+
Type: "Ready-In-Zone-zone1",
616+
Status: metav1.ConditionTrue,
617+
Reason: "SetupSucceeded",
618+
Message: "NetworkQoS Destinations applied",
619+
}, {
620+
Type: "Ready-In-Zone-zone2",
621+
Status: metav1.ConditionTrue,
622+
Reason: "SetupSucceeded",
623+
Message: "NetworkQoS Destinations applied",
624+
}},
625+
}, fakeClient)
626+
checkNQStatusEventually(networkQoS, false, false, fakeClient)
627+
628+
})
629+
630+
It("updates NetworkQoS status with UnknownZone", func() {
631+
config.OVNKubernetesFeature.EnableNetworkQoS = true
632+
zones := sets.New[string]("zone1", zone_tracker.UnknownZone)
633+
namespace1 := util.NewNamespace(namespace1Name)
634+
networkQoS := newNetworkQoS(namespace1.Name)
635+
start(zones, namespace1, networkQoS)
636+
637+
// no matter how many messages are in the status, it won't be updated while UnknownZone is present
638+
updateNetworkQoSStatus(networkQoS, &networkqosapi.Status{
639+
Conditions: []metav1.Condition{{
640+
Type: "Ready-In-Zone-zone1",
641+
Status: metav1.ConditionTrue,
642+
Reason: "SetupSucceeded",
643+
Message: "NetworkQoS Destinations applied",
644+
}},
645+
}, fakeClient)
646+
checkEmptyNQStatusConsistently(networkQoS, fakeClient)
647+
648+
// when UnknownZone is removed, updates will be handled, but status from the new zone is not reported yet
649+
statusManager.onZoneUpdate(sets.New[string]("zone1", "zone2"))
650+
checkEmptyNQStatusConsistently(networkQoS, fakeClient)
651+
// when new zone status is reported, status will be set
652+
updateNetworkQoSStatus(networkQoS, &networkqosapi.Status{
653+
Conditions: []metav1.Condition{{
654+
Type: "Ready-In-Zone-zone1",
655+
Status: metav1.ConditionTrue,
656+
Reason: "SetupSucceeded",
657+
Message: "NetworkQoS Destinations applied",
658+
}, {
659+
Type: "Ready-In-Zone-zone2",
660+
Status: metav1.ConditionTrue,
661+
Reason: "SetupSucceeded",
662+
Message: "NetworkQoS Destinations applied",
663+
}},
664+
}, fakeClient)
665+
checkNQStatusEventually(networkQoS, false, false, fakeClient)
666+
})
667+
508668
})

‎go-controller/pkg/config/config.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -432,6 +432,7 @@ type OVNKubernetesFeatureConfig struct {
432432
EnableDNSNameResolver bool `gcfg:"enable-dns-name-resolver"`
433433
EnableServiceTemplateSupport bool `gcfg:"enable-svc-template-support"`
434434
EnableObservability bool `gcfg:"enable-observability"`
435+
EnableNetworkQoS bool `gcfg:"enable-network-qos"`
435436
}
436437

437438
// GatewayMode holds the node gateway mode

‎go-controller/pkg/controllermanager/controller_manager.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,7 @@ func NewControllerManager(ovnClient *util.OVNClientset, wf *factory.WatchFactory
210210
APBRouteClient: ovnClient.AdminPolicyRouteClient,
211211
EgressQoSClient: ovnClient.EgressQoSClient,
212212
IPAMClaimsClient: ovnClient.IPAMClaimsClient,
213+
NetworkQoSClient: ovnClient.NetworkQoSClient,
213214
},
214215
stopChan: stopCh,
215216
watchFactory: wf,

‎go-controller/pkg/factory/factory.go

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,7 @@ func (wf *WatchFactory) ShallowClone() *WatchFactory {
151151
udnFactory: wf.udnFactory,
152152
raFactory: wf.raFactory,
153153
frrFactory: wf.frrFactory,
154+
networkQoSFactory: wf.networkQoSFactory,
154155
informers: wf.informers,
155156
stopChan: wf.stopChan,
156157

@@ -514,7 +515,8 @@ func NewOVNKubeControllerWatchFactory(ovnClientset *util.OVNKubeControllerClient
514515
}
515516

516517
if config.OVNKubernetesFeature.EnableNetworkQoS {
517-
wf.informers[NetworkQoSType], err = newInformer(NetworkQoSType, wf.networkQoSFactory.K8s().V1().NetworkQoSes().Informer())
518+
wf.informers[NetworkQoSType], err = newQueuedInformer(eventQueueSize, NetworkQoSType,
519+
wf.networkQoSFactory.K8s().V1().NetworkQoSes().Informer(), wf.stopChan, minNumEventQueues)
518520
if err != nil {
519521
return nil, err
520522
}
@@ -627,6 +629,15 @@ func (wf *WatchFactory) Start() error {
627629
}
628630
}
629631

632+
if config.OVNKubernetesFeature.EnableNetworkQoS && wf.networkQoSFactory != nil {
633+
wf.networkQoSFactory.Start(wf.stopChan)
634+
for oType, synced := range waitForCacheSyncWithTimeout(wf.networkQoSFactory, wf.stopChan) {
635+
if !synced {
636+
return fmt.Errorf("error in syncing cache for %v informer", oType)
637+
}
638+
}
639+
}
640+
630641
if util.IsNetworkSegmentationSupportEnabled() && wf.udnFactory != nil {
631642
wf.udnFactory.Start(wf.stopChan)
632643
for oType, synced := range waitForCacheSyncWithTimeout(wf.udnFactory, wf.stopChan) {
@@ -1181,6 +1192,10 @@ func getObjectMeta(objType reflect.Type, obj interface{}) (*metav1.ObjectMeta, e
11811192
if cudn, ok := obj.(*userdefinednetworkapi.ClusterUserDefinedNetwork); ok {
11821193
return &cudn.ObjectMeta, nil
11831194
}
1195+
case NetworkQoSType:
1196+
if networkQoS, ok := obj.(*networkqosapi.NetworkQoS); ok {
1197+
return &networkQoS.ObjectMeta, nil
1198+
}
11841199
}
11851200

11861201
return nil, fmt.Errorf("cannot get ObjectMeta from type %v", objType)

‎go-controller/pkg/factory/factory_test.go

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -230,11 +230,17 @@ func newNetworkQoS(name, namespace string) *networkqos.NetworkQoS {
230230
return &networkqos.NetworkQoS{
231231
ObjectMeta: newObjectMeta(name, namespace),
232232
Spec: networkqos.Spec{
233-
NetworkAttachmentName: "default/stream",
233+
NetworkAttachmentRefs: []v1.ObjectReference{
234+
{
235+
Kind: "NetworkAttachmentDefinition",
236+
Namespace: "default",
237+
Name: "stream",
238+
},
239+
},
240+
Priority: 100,
234241
Egress: []networkqos.Rule{
235242
{
236-
Priority: 100,
237-
DSCP: 50,
243+
DSCP: 50,
238244
Classifier: networkqos.Classifier{
239245
To: []networkqos.Destination{
240246
{

0 commit comments

Comments
 (0)