Merge pull request #5192 from tssurya/bgp-isolation-udnpod-to-default-net-svc-take2

Isolate default network services from UDN pods

Author: tssurya

go-controller/pkg/node/gateway.go

@@ -241,7 +241,6 @@ func (g *gateway) AddEgressIP(eip *egressipv1.EgressIP) error {
 		if err = g.Reconcile(); err != nil {
 			return fmt.Errorf("failed to sync gateway: %v", err)
 		}
-		g.openflowManager.requestFlowSync()
 	}
 	return nil
 }
@@ -258,7 +257,6 @@ func (g *gateway) UpdateEgressIP(oldEIP, newEIP *egressipv1.EgressIP) error {
 		if err = g.Reconcile(); err != nil {
 			return fmt.Errorf("failed to sync gateway: %v", err)
 		}
-		g.openflowManager.requestFlowSync()
 	}
 	return nil
 }
@@ -275,7 +273,6 @@ func (g *gateway) DeleteEgressIP(eip *egressipv1.EgressIP) error {
 		if err = g.Reconcile(); err != nil {
 			return fmt.Errorf("failed to sync gateway: %v", err)
 		}
-		g.openflowManager.requestFlowSync()
 	}
 	return nil
 }
@@ -290,7 +287,6 @@ func (g *gateway) SyncEgressIP(eips []interface{}) error {
 	if err := g.Reconcile(); err != nil {
 		return fmt.Errorf("failed to sync gateway: %v", err)
 	}
-	g.openflowManager.requestFlowSync()
 	return nil
 }
 
@@ -492,6 +488,8 @@ func (g *gateway) Reconcile() error {
 	if err := g.openflowManager.updateBridgeFlowCache(g.nodeIPManager.ListAddresses()); err != nil {
 		return err
 	}
+	// let's sync these flows immediately
+	g.openflowManager.requestFlowSync()
 	err := g.updateSNATRules()
 	if err != nil {
 		return err

go-controller/pkg/node/gateway_shared_intf.go

@@ -411,18 +411,33 @@ func (npw *nodePortWatcher) updateServiceFlowCache(service *corev1.Service, netI
 			}
 
 			ipPrefix := "ip"
-			masqueradeSubnet := config.Gateway.V4MasqueradeSubnet
 			if !utilnet.IsIPv4String(service.Spec.ClusterIP) {
 				ipPrefix = "ipv6"
-				masqueradeSubnet = config.Gateway.V6MasqueradeSubnet
 			}
 			// table 2, user-defined network host -> OVN towards default cluster network services
 			defaultNetConfig := npw.ofm.defaultBridge.getActiveNetworkBridgeConfig(types.DefaultNetworkName)
-
-			npw.ofm.updateFlowCacheEntry(key, []string{fmt.Sprintf("cookie=%s, priority=300, table=2, %s, %s_src=%s, %s_dst=%s, "+
+			// sample flow: cookie=0xdeff105, duration=2319.685s, table=2, n_packets=496, n_bytes=67111, priority=300,
+			//              ip,nw_dst=10.96.0.1 actions=mod_dl_dst:02:42:ac:12:00:03,output:"patch-breth0_ov"
+			// This flow is used for UDNs and advertised UDNs to be able to reach kapi and dns services alone on default network
+			flows := []string{fmt.Sprintf("cookie=%s, priority=300, table=2, %s, %s_dst=%s, "+
 				"actions=set_field:%s->eth_dst,output:%s",
-				defaultOpenFlowCookie, ipPrefix, ipPrefix, masqueradeSubnet, ipPrefix, service.Spec.ClusterIP,
-				npw.ofm.getDefaultBridgeMAC().String(), defaultNetConfig.ofPortPatch)})
+				defaultOpenFlowCookie, ipPrefix, ipPrefix, service.Spec.ClusterIP,
+				npw.ofm.getDefaultBridgeMAC().String(), defaultNetConfig.ofPortPatch)}
+			if util.IsRouteAdvertisementsEnabled() {
+				// if the network is advertised, then for the reply from kapi and dns services to go back
+				// into the UDN's VRF we need flows that statically send this to the local port
+				// sample flow: cookie=0xdeff105, duration=264.196s, table=0, n_packets=0, n_bytes=0, priority=490,ip,
+				//              in_port="patch-breth0_ov",nw_src=10.96.0.10,actions=ct(table=3,zone=64001,nat)
+				// this flow is meant to match all advertised UDNs and then the ip rules on the host will take
+				// this packet into the corresponding UDNs
+				// NOTE: We chose priority 490 to differentiate this flow from the flow at priority 500 added for the
+				// non-advertised UDNs reponse for debugging purposes:
+				// sample flow for non-advertised UDNs: cookie=0xdeff105, duration=684.087s, table=0, n_packets=0, n_bytes=0,
+				//				idle_age=684, priority=500,ip,in_port=2,nw_src=10.96.0.0/16,nw_dst=169.254.0.0/17 actions=ct(table=3,zone=64001,nat)
+				flows = append(flows, fmt.Sprintf("cookie=%s, priority=490, in_port=%s, ip, ip_src=%s,actions=ct(zone=%d,nat,table=3)",
+					defaultOpenFlowCookie, defaultNetConfig.ofPortPatch, service.Spec.ClusterIP, config.Default.HostMasqConntrackZone))
+			}
+			npw.ofm.updateFlowCacheEntry(key, flows)
 		}
 	}
 	return utilerrors.Join(errors...)
@@ -1593,6 +1608,37 @@ func flowsForDefaultBridge(bridge *bridgeConfiguration, extraIPs []net.IP) ([]st
 					"actions=ct(commit,zone=%d,table=2)",
 					defaultOpenFlowCookie, ofPortHost, protoPrefix, protoPrefix,
 					masqSubnet, protoPrefix, svcCIDR, config.Default.HostMasqConntrackZone))
+			if util.IsRouteAdvertisementsEnabled() {
+				// If the UDN is advertised then instead of matching on the masqSubnet
+				// we match on the UDNPodSubnet itself and we also don't SNAT to 169.254.0.2
+				// sample flow: cookie=0xdeff105, duration=1472.742s, table=0, n_packets=9, n_bytes=666, priority=550
+				//              ip,in_port=LOCAL,nw_src=103.103.0.0/16,nw_dst=10.96.0.0/16 actions=ct(commit,table=2,zone=64001)
+				for _, netConfig := range bridge.patchedNetConfigs() {
+					if netConfig.isDefaultNetwork() {
+						continue
+					}
+					if netConfig.advertised.Load() {
+						var udnAdvertisedSubnets []*net.IPNet
+						for _, clusterEntry := range netConfig.subnets {
+							udnAdvertisedSubnets = append(udnAdvertisedSubnets, clusterEntry.CIDR)
+						}
+						// Filter subnets based on the clusterIP service family
+						// NOTE: We don't support more than 1 subnet CIDR of same family type; we only pick the first one
+						matchingIPFamilySubnet, err := util.MatchFirstIPNetFamily(utilnet.IsIPv6CIDR(svcCIDR), udnAdvertisedSubnets)
+						if err != nil {
+							klog.Infof("Unable to determine UDN subnet for the provided family isIPV6: %t, %v", utilnet.IsIPv6CIDR(svcCIDR), err)
+							continue
+						}
+
+						// Use the filtered subnet for the flow compute instead of the masqueradeIP
+						dftFlows = append(dftFlows,
+							fmt.Sprintf("cookie=%s, priority=550, in_port=%s, %s, %s_src=%s, %s_dst=%s, "+
+								"actions=ct(commit,zone=%d,table=2)",
+								defaultOpenFlowCookie, ofPortHost, protoPrefix, protoPrefix,
+								matchingIPFamilySubnet.String(), protoPrefix, svcCIDR, config.Default.HostMasqConntrackZone))
+					}
+				}
+			}
 		}
 
 		masqDst := masqIP
@@ -1706,10 +1752,27 @@ func flowsForDefaultBridge(bridge *bridgeConfiguration, extraIPs []net.IP) ([]st
 			if netConfig.isDefaultNetwork() {
 				continue
 			}
+			srcIPOrSubnet := netConfig.v4MasqIPs.ManagementPort.IP.String()
+			if util.IsRouteAdvertisementsEnabled() && netConfig.advertised.Load() {
+				var udnAdvertisedSubnets []*net.IPNet
+				for _, clusterEntry := range netConfig.subnets {
+					udnAdvertisedSubnets = append(udnAdvertisedSubnets, clusterEntry.CIDR)
+				}
+				// Filter subnets based on the clusterIP service family
+				// NOTE: We don't support more than 1 subnet CIDR of same family type; we only pick the first one
+				matchingIPFamilySubnet, err := util.MatchFirstIPNetFamily(false, udnAdvertisedSubnets)
+				if err != nil {
+					klog.Infof("Unable to determine IPV4 UDN subnet for the provided family isIPV6: %v", err)
+					continue
+				}
+
+				// Use the filtered subnets for the flow compute instead of the masqueradeIP
+				srcIPOrSubnet = matchingIPFamilySubnet.String()
+			}
 			dftFlows = append(dftFlows,
 				fmt.Sprintf("cookie=%s, priority=200, table=2, ip, ip_src=%s, "+
 					"actions=set_field:%s->eth_dst,output:%s",
-					defaultOpenFlowCookie, netConfig.v4MasqIPs.ManagementPort.IP,
+					defaultOpenFlowCookie, srcIPOrSubnet,
 					bridgeMacAddress, netConfig.ofPortPatch))
 			dftFlows = append(dftFlows,
 				fmt.Sprintf("cookie=%s, priority=200, table=2, ip, pkt_mark=%s, "+
@@ -1724,11 +1787,27 @@ func flowsForDefaultBridge(bridge *bridgeConfiguration, extraIPs []net.IP) ([]st
 			if netConfig.isDefaultNetwork() {
 				continue
 			}
+			srcIPOrSubnet := netConfig.v6MasqIPs.ManagementPort.IP.String()
+			if util.IsRouteAdvertisementsEnabled() && netConfig.advertised.Load() {
+				var udnAdvertisedSubnets []*net.IPNet
+				for _, clusterEntry := range netConfig.subnets {
+					udnAdvertisedSubnets = append(udnAdvertisedSubnets, clusterEntry.CIDR)
+				}
+				// Filter subnets based on the clusterIP service family
+				// NOTE: We don't support more than 1 subnet CIDR of same family type; we only pick the first one
+				matchingIPFamilySubnet, err := util.MatchFirstIPNetFamily(true, udnAdvertisedSubnets)
+				if err != nil {
+					klog.Infof("Unable to determine IPV6 UDN subnet for the provided family isIPV6: %v", err)
+					continue
+				}
 
+				// Use the filtered subnets for the flow compute instead of the masqueradeIP
+				srcIPOrSubnet = matchingIPFamilySubnet.String()
+			}
 			dftFlows = append(dftFlows,
 				fmt.Sprintf("cookie=%s, priority=200, table=2, ip6, ipv6_src=%s, "+
 					"actions=set_field:%s->eth_dst,output:%s",
-					defaultOpenFlowCookie, netConfig.v6MasqIPs.ManagementPort.IP,
+					defaultOpenFlowCookie, srcIPOrSubnet,
 					bridgeMacAddress, netConfig.ofPortPatch))
 			dftFlows = append(dftFlows,
 				fmt.Sprintf("cookie=%s, priority=200, table=2, ip6, pkt_mark=%s, "+

go-controller/pkg/node/gateway_udn.go

@@ -932,9 +932,12 @@ func (udng *UserDefinedNetworkGateway) doReconcile() error {
 	// add below OpenFlows based on the gateway mode and whether the network is advertised or not:
 	// table=1, n_packets=0, n_bytes=0, priority=16,ip,nw_dst=128.192.0.2 actions=LOCAL (Both gateway modes)
 	// table=1, n_packets=0, n_bytes=0, priority=15,ip,nw_dst=128.192.0.0/14 actions=output:3 (shared gateway mode)
+	// necessary service isolation flows based on whether network is advertised or not
 	if err := udng.openflowManager.updateBridgeFlowCache(udng.nodeIPManager.ListAddresses()); err != nil {
 		return fmt.Errorf("error while updating logical flow for UDN %s: %s", udng.GetNetworkName(), err)
 	}
+	// let's sync these flows immediately
+	udng.openflowManager.requestFlowSync()
 
 	if err := udng.updateAdvertisedUDNIsolationRules(isNetworkAdvertised); err != nil {
 		return fmt.Errorf("error while updating advertised UDN isolation rules for network %s: %w", udng.GetNetworkName(), err)