Skip to content

Commit b057704

Browse files
danwinshipdanwinship
committed
Make egress IP code handle node IP address changes
1 parent 4815e18 commit b057704

File tree

3 files changed

+149
-8
lines changed

3 files changed

+149
-8
lines changed

‎pkg/network/common/egressip.go

Lines changed: 36 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88

99
"github.com/golang/glog"
1010

11+
ktypes "k8s.io/apimachinery/pkg/types"
1112
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
1213
"k8s.io/apimachinery/pkg/util/sets"
1314
"k8s.io/apimachinery/pkg/watch"
@@ -63,6 +64,7 @@ type EgressIPTracker struct {
6364

6465
watcher EgressIPWatcher
6566

67+
nodes map[ktypes.UID]*nodeEgress
6668
nodesByNodeIP map[string]*nodeEgress
6769
namespacesByVNID map[uint32]*namespaceEgress
6870
egressIPs map[string]*egressIPInfo
@@ -77,6 +79,7 @@ func NewEgressIPTracker(watcher EgressIPWatcher) *EgressIPTracker {
7779
return &EgressIPTracker{
7880
watcher: watcher,
7981

82+
nodes: make(map[ktypes.UID]*nodeEgress),
8083
nodesByNodeIP: make(map[string]*nodeEgress),
8184
namespacesByVNID: make(map[uint32]*namespaceEgress),
8285
egressIPs: make(map[string]*egressIPInfo),
@@ -167,9 +170,10 @@ func (eit *EgressIPTracker) handleDeleteHostSubnet(obj interface{}) {
167170
hs := obj.(*networkapi.HostSubnet)
168171
glog.V(5).Infof("Watch %s event for HostSubnet %q", watch.Deleted, hs.Name)
169172

170-
eit.UpdateHostSubnetEgress(&networkapi.HostSubnet{
171-
HostIP: hs.HostIP,
172-
})
173+
hs = hs.DeepCopy()
174+
hs.EgressCIDRs = nil
175+
hs.EgressIPs = nil
176+
eit.UpdateHostSubnetEgress(hs)
173177
}
174178

175179
func (eit *EgressIPTracker) UpdateHostSubnetEgress(hs *networkapi.HostSubnet) {
@@ -185,7 +189,7 @@ func (eit *EgressIPTracker) UpdateHostSubnetEgress(hs *networkapi.HostSubnet) {
185189
sdnIP = netutils.GenerateDefaultGateway(cidr).String()
186190
}
187191

188-
node := eit.nodesByNodeIP[hs.HostIP]
192+
node := eit.nodes[hs.UID]
189193
if node == nil {
190194
if len(hs.EgressIPs) == 0 && len(hs.EgressCIDRs) == 0 {
191195
return
@@ -196,9 +200,11 @@ func (eit *EgressIPTracker) UpdateHostSubnetEgress(hs *networkapi.HostSubnet) {
196200
sdnIP: sdnIP,
197201
requestedIPs: sets.NewString(),
198202
}
203+
eit.nodes[hs.UID] = node
199204
eit.nodesByNodeIP[hs.HostIP] = node
200205
} else if len(hs.EgressIPs) == 0 && len(hs.EgressCIDRs) == 0 {
201-
delete(eit.nodesByNodeIP, hs.HostIP)
206+
delete(eit.nodes, hs.UID)
207+
delete(eit.nodesByNodeIP, node.nodeIP)
202208
}
203209

204210
// Process EgressCIDRs
@@ -218,6 +224,28 @@ func (eit *EgressIPTracker) UpdateHostSubnetEgress(hs *networkapi.HostSubnet) {
218224
eit.updateEgressCIDRs = true
219225
}
220226

227+
if node.nodeIP != hs.HostIP {
228+
// We have to clean up the old egress IP mappings and call syncEgressIPs
229+
// before we can change node.nodeIP
230+
movedEgressIPs := make([]string, 0, node.requestedIPs.Len())
231+
for _, ip := range node.requestedIPs.UnsortedList() {
232+
eg := eit.egressIPs[ip]
233+
if eg.assignedNodeIP == node.nodeIP {
234+
movedEgressIPs = append(movedEgressIPs, ip)
235+
eit.deleteNodeEgressIP(node, ip)
236+
}
237+
}
238+
eit.syncEgressIPs()
239+
240+
delete(eit.nodesByNodeIP, node.nodeIP)
241+
node.nodeIP = hs.HostIP
242+
eit.nodesByNodeIP[node.nodeIP] = node
243+
244+
for _, ip := range movedEgressIPs {
245+
eit.addNodeEgressIP(node, ip)
246+
}
247+
}
248+
221249
// Process new and removed EgressIPs
222250
oldRequestedIPs := node.requestedIPs
223251
node.requestedIPs = sets.NewString(hs.EgressIPs...)
@@ -448,7 +476,7 @@ func (eit *EgressIPTracker) findEgressIPAllocation(ip net.IP, allocation map[str
448476
bestNode := ""
449477
otherNodes := false
450478

451-
for _, node := range eit.nodesByNodeIP {
479+
for _, node := range eit.nodes {
452480
egressIPs, exists := allocation[node.nodeName]
453481
if !exists {
454482
continue
@@ -478,7 +506,7 @@ func (eit *EgressIPTracker) ReallocateEgressIPs() map[string][]string {
478506
allocation := make(map[string][]string)
479507
changed := make(map[string]bool)
480508
alreadyAllocated := make(map[string]bool)
481-
for _, node := range eit.nodesByNodeIP {
509+
for _, node := range eit.nodes {
482510
if len(node.parsedCIDRs) > 0 {
483511
allocation[node.nodeName] = make([]string, 0, node.requestedIPs.Len())
484512
}
@@ -534,7 +562,7 @@ func (eit *EgressIPTracker) ReallocateEgressIPs() map[string][]string {
534562
}
535563

536564
// Remove unchanged nodes from the return value
537-
for _, node := range eit.nodesByNodeIP {
565+
for _, node := range eit.nodes {
538566
if !changed[node.nodeName] {
539567
delete(allocation, node.nodeName)
540568
}

‎pkg/network/common/egressip_test.go

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -947,3 +947,57 @@ func TestEgressCIDRAllocation(t *testing.T) {
947947
t.Fatalf("%v", err)
948948
}
949949
}
950+
951+
func TestEgressNodeRenumbering(t *testing.T) {
952+
eit, w := setupEgressIPTracker(t)
953+
954+
updateHostSubnetEgress(eit, &networkapi.HostSubnet{
955+
Host: "alpha",
956+
HostIP: "172.17.0.3",
957+
EgressIPs: []string{"172.17.0.100"},
958+
})
959+
updateHostSubnetEgress(eit, &networkapi.HostSubnet{
960+
Host: "beta",
961+
HostIP: "172.17.0.4",
962+
EgressIPs: []string{"172.17.0.101"},
963+
})
964+
updateHostSubnetEgress(eit, &networkapi.HostSubnet{
965+
Host: "gamma",
966+
HostIP: "172.17.0.5",
967+
EgressIPs: []string{"172.17.0.102"},
968+
})
969+
updateNetNamespaceEgress(eit, &networkapi.NetNamespace{
970+
NetID: 42,
971+
EgressIPs: []string{"172.17.0.100"},
972+
})
973+
updateNetNamespaceEgress(eit, &networkapi.NetNamespace{
974+
NetID: 43,
975+
EgressIPs: []string{"172.17.0.101"},
976+
})
977+
978+
err := w.assertChanges(
979+
"claim 172.17.0.100 on 172.17.0.3 for namespace 42",
980+
"namespace 42 via 172.17.0.100 on 172.17.0.3",
981+
"claim 172.17.0.101 on 172.17.0.4 for namespace 43",
982+
"namespace 43 via 172.17.0.101 on 172.17.0.4",
983+
)
984+
if err != nil {
985+
t.Fatalf("%v", err)
986+
}
987+
988+
// Renumber one of the hosts
989+
updateHostSubnetEgress(eit, &networkapi.HostSubnet{
990+
Host: "beta",
991+
HostIP: "172.17.0.6",
992+
EgressIPs: []string{"172.17.0.101"},
993+
})
994+
err = w.assertChanges(
995+
"release 172.17.0.101 on 172.17.0.4",
996+
"namespace 43 dropped",
997+
"claim 172.17.0.101 on 172.17.0.6 for namespace 43",
998+
"namespace 43 via 172.17.0.101 on 172.17.0.6",
999+
)
1000+
if err != nil {
1001+
t.Fatalf("%v", err)
1002+
}
1003+
}

‎pkg/network/node/egressip_test.go

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -722,3 +722,62 @@ func TestMarkForVNID(t *testing.T) {
722722
}
723723
}
724724
}
725+
726+
func TestEgressNodeRenumbering(t *testing.T) {
727+
eip, flows := setupEgressIPWatcher(t)
728+
729+
eip.tracker.UpdateHostSubnetEgress(&networkapi.HostSubnet{
730+
ObjectMeta: metav1.ObjectMeta{
731+
Name: "alpha",
732+
UID: ktypes.UID("alpha"),
733+
},
734+
Host: "alpha",
735+
HostIP: "172.17.0.3",
736+
EgressIPs: []string{"172.17.0.100"},
737+
})
738+
eip.tracker.UpdateHostSubnetEgress(&networkapi.HostSubnet{
739+
ObjectMeta: metav1.ObjectMeta{
740+
Name: "beta",
741+
UID: ktypes.UID("beta"),
742+
},
743+
Host: "beta",
744+
HostIP: "172.17.0.4",
745+
EgressIPs: []string{"172.17.0.101"},
746+
})
747+
eip.tracker.UpdateHostSubnetEgress(&networkapi.HostSubnet{
748+
ObjectMeta: metav1.ObjectMeta{
749+
Name: "gamma",
750+
UID: ktypes.UID("gamma"),
751+
},
752+
Host: "gamma",
753+
HostIP: "172.17.0.5",
754+
EgressIPs: []string{"172.17.0.102"},
755+
})
756+
updateNamespaceEgress(eip, 42, []string{"172.17.0.100"})
757+
updateNamespaceEgress(eip, 43, []string{"172.17.0.101"})
758+
759+
err := assertOVSChanges(eip, &flows,
760+
egressOVSChange{vnid: 42, egress: Remote, remote: "172.17.0.3"},
761+
egressOVSChange{vnid: 43, egress: Local},
762+
)
763+
if err != nil {
764+
t.Fatalf("%v", err)
765+
}
766+
767+
// Renumber one of the hosts
768+
eip.tracker.UpdateHostSubnetEgress(&networkapi.HostSubnet{
769+
ObjectMeta: metav1.ObjectMeta{
770+
Name: "beta",
771+
UID: ktypes.UID("beta"),
772+
},
773+
Host: "beta",
774+
HostIP: "172.17.0.6",
775+
EgressIPs: []string{"172.17.0.101"},
776+
})
777+
err = assertOVSChanges(eip, &flows,
778+
egressOVSChange{vnid: 43, egress: Remote, remote: "172.17.0.6"},
779+
)
780+
if err != nil {
781+
t.Fatalf("%v", err)
782+
}
783+
}

0 commit comments

Comments
 (0)