sdn: rationalize data directories between kubelet, CNI, and SDN

Instead of defining things in a couple places, just use the Kubelet
CNI driver conf dir define.  If that's overridden via the kubelet
--cni-conf-dir argument, use that instead.

Also pass a known IPAM data dir to the host-local plugin, and use
that in the metrics code, to protect against upstream changes to
the defaults.

Author: dcbw

pkg/cmd/server/kubernetes/network/sdn_linux.go

@@ -10,6 +10,7 @@ import (
 	"k8s.io/client-go/tools/record"
 	kinternalclientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
 	kinternalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
+	kcni "k8s.io/kubernetes/pkg/kubelet/network/cni"
 	"k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig"
 
 	configapi "github.com/openshift/origin/pkg/cmd/server/api"
@@ -35,6 +36,11 @@ func NewSDNInterfaces(options configapi.NodeConfig, networkClient networkclient.
 		}
 	}
 
+	cniConfDir := kcni.DefaultNetDir
+	if val, ok := options.KubeletArguments["cni-conf-dir"]; ok && len(val) == 1 {
+		cniConfDir = val[0]
+	}
+
 	// dockershim + kube CNI driver delegates hostport handling to plugins,
 	// while CRI-O handles hostports itself. Thus we need to disable the
 	// SDN's hostport handling when run under CRI-O.
@@ -49,6 +55,7 @@ func NewSDNInterfaces(options configapi.NodeConfig, networkClient networkclient.
 		Hostname:           options.NodeName,
 		SelfIP:             options.NodeIP,
 		RuntimeEndpoint:    runtimeEndpoint,
+		CNIConfDir:         cniConfDir,
 		MTU:                options.NetworkConfig.MTU,
 		NetworkClient:      networkClient,
 		KClient:            kubeClient,

pkg/network/node/metrics.go

@@ -167,7 +167,7 @@ func updateARPMetrics() {
 
 func updatePodIPMetrics() {
 	numAddrs := 0
-	items, err := ioutil.ReadDir("/var/lib/cni/networks/openshift-sdn/")
+	items, err := ioutil.ReadDir(hostLocalDataDir + "/networks/openshift-sdn/")
 	if err != nil && os.IsNotExist(err) {
 		// Don't log an error if the directory doesn't exist (eg, no pods started yet)
 		return

pkg/network/node/node.go

@@ -47,8 +47,8 @@ import (
 )
 
 const (
-	cniDirPath       = "/etc/cni/net.d"
 	openshiftCNIFile = "80-openshift-network.conf"
+	hostLocalDataDir = "/var/lib/cni"
 )
 
 type osdnPolicy interface {
@@ -75,6 +75,7 @@ type OsdnNodeConfig struct {
 	RuntimeEndpoint string
 	MTU             uint32
 	EnableHostports bool
+	CNIConfDir      string
 
 	NetworkClient networkclient.Interface
 	KClient       kclientset.Interface
@@ -102,6 +103,7 @@ type OsdnNode struct {
 	useConnTrack       bool
 	iptablesSyncPeriod time.Duration
 	mtu                uint32
+	cniDirPath         string
 
 	// Synchronizes operations on egressPolicies
 	egressPoliciesLock sync.Mutex
@@ -154,7 +156,7 @@ func New(c *OsdnNodeConfig) (network.NodeInterface, error) {
 
 	// If our CNI config file exists, remove it so that kubelet doesn't think
 	// we're ready yet
-	os.Remove(filepath.Join(cniDirPath, openshiftCNIFile))
+	os.Remove(filepath.Join(c.CNIConfDir, openshiftCNIFile))
 
 	if err := c.setNodeIP(); err != nil {
 		return nil, err
@@ -184,6 +186,7 @@ func New(c *OsdnNodeConfig) (network.NodeInterface, error) {
 		kubeInformers:      c.KubeInformers,
 		networkInformers:   c.NetworkInformers,
 		egressIP:           newEgressIPWatcher(oc, c.SelfIP, c.MasqueradeBit),
+		cniDirPath:         c.CNIConfDir,
 
 		runtimeEndpoint: c.RuntimeEndpoint,
 		// 2 minutes is the current default value used in kubelet
@@ -380,7 +383,7 @@ func (node *OsdnNode) Start() error {
 		}
 	}
 
-	if err := os.MkdirAll(cniDirPath, 0755); err != nil {
+	if err := os.MkdirAll(node.cniDirPath, 0755); err != nil {
 		return err
 	}
 
@@ -396,7 +399,7 @@ func (node *OsdnNode) Start() error {
 
 	// Write our CNI config file out to disk to signal to kubelet that
 	// our network plugin is ready
-	return ioutil.WriteFile(filepath.Join(cniDirPath, openshiftCNIFile), []byte(`
+	return ioutil.WriteFile(filepath.Join(node.cniDirPath, openshiftCNIFile), []byte(`
 {
   "cniVersion": "0.2.0",
   "name": "openshift-sdn",

pkg/network/node/pod.go

@@ -26,6 +26,7 @@ import (
 	kclientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
 	kcontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	knetwork "k8s.io/kubernetes/pkg/kubelet/network"
+	kcni "k8s.io/kubernetes/pkg/kubelet/network/cni"
 	kubehostport "k8s.io/kubernetes/pkg/kubelet/network/hostport"
 	kbandwidth "k8s.io/kubernetes/pkg/util/bandwidth"
 	utildbus "k8s.io/kubernetes/pkg/util/dbus"
@@ -42,6 +43,7 @@ import (
 
 const (
 	podInterfaceName = knetwork.DefaultInterfaceName
+	cniBinPath       = kcni.DefaultCNIDir
 )
 
 type podHandler interface {
@@ -114,9 +116,10 @@ func getIPAMConfig(clusterNetworks []common.ClusterNetwork, localSubnet string)
 	}
 
 	type hostLocalIPAM struct {
-		Type   string           `json:"type"`
-		Subnet cnitypes.IPNet   `json:"subnet"`
-		Routes []cnitypes.Route `json:"routes"`
+		Type    string           `json:"type"`
+		Subnet  cnitypes.IPNet   `json:"subnet"`
+		Routes  []cnitypes.Route `json:"routes"`
+		DataDir string           `json:"dataDir"`
 	}
 
 	type cniNetworkConfig struct {
@@ -153,7 +156,8 @@ func getIPAMConfig(clusterNetworks []common.ClusterNetwork, localSubnet string)
 		Name:       "openshift-sdn",
 		Type:       "openshift-sdn",
 		IPAM: &hostLocalIPAM{
-			Type: "host-local",
+			Type:    "host-local",
+			DataDir: hostLocalDataDir,
 			Subnet: cnitypes.IPNet{
 				IP:   nodeNet.IP,
 				Mask: nodeNet.Mask,
@@ -410,7 +414,7 @@ func createIPAMArgs(netnsPath string, action cniserver.CNICommand, id string) *i
 		ContainerID: id,
 		NetNS:       netnsPath,
 		IfName:      podInterfaceName,
-		Path:        "/opt/cni/bin",
+		Path:        cniBinPath,
 	}
 }
 
@@ -421,7 +425,7 @@ func (m *podManager) ipamAdd(netnsPath string, id string) (*cni020.Result, net.I
 	}
 
 	args := createIPAMArgs(netnsPath, cniserver.CNI_ADD, id)
-	r, err := invoke.ExecPluginWithResult("/opt/cni/bin/host-local", m.ipamConfig, args)
+	r, err := invoke.ExecPluginWithResult(cniBinPath+"/host-local", m.ipamConfig, args)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to run CNI IPAM ADD: %v", err)
 	}
@@ -441,7 +445,7 @@ func (m *podManager) ipamAdd(netnsPath string, id string) (*cni020.Result, net.I
 // Run CNI IPAM release for the container
 func (m *podManager) ipamDel(id string) error {
 	args := createIPAMArgs("", cniserver.CNI_DEL, id)
-	err := invoke.ExecPluginWithoutResult("/opt/cni/bin/host-local", m.ipamConfig, args)
+	err := invoke.ExecPluginWithoutResult(cniBinPath+"/host-local", m.ipamConfig, args)
 	if err != nil {
 		return fmt.Errorf("failed to run CNI IPAM DEL: %v", err)
 	}