Skip to content

Commit 9740cf7

Browse files
Network component should refresh certificates if they expire
A single process openshift start node has both kubelet and network. Kubelet rotates its client certs - network does not. Until we split out network we need to do something minimal to ensure the cert is refreshed. Use the same code as the kubelet, but when the cert expires check disk and see if it was refreshed. That should work for bootstrapped environments because the file is updated.
1 parent 7968b96 commit 9740cf7

File tree

2 files changed

+239
-0
lines changed

2 files changed

+239
-0
lines changed

pkg/cmd/server/kubernetes/network/network_config.go

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,22 @@ package network
33
import (
44
"fmt"
55
"net"
6+
"time"
67

78
miekgdns "github.com/miekg/dns"
89

10+
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
11+
utilwait "k8s.io/apimachinery/pkg/util/wait"
912
kclientset "k8s.io/client-go/kubernetes"
13+
"k8s.io/client-go/rest"
1014
"k8s.io/kubernetes/pkg/apis/componentconfig"
1115
kclientsetexternal "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
1216
kclientsetinternal "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
1317
kinternalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
18+
"k8s.io/kubernetes/pkg/kubelet/certificate"
1419

1520
configapi "github.com/openshift/origin/pkg/cmd/server/api"
21+
"github.com/openshift/origin/pkg/cmd/server/kubernetes/network/transport"
1622
"github.com/openshift/origin/pkg/dns"
1723
"github.com/openshift/origin/pkg/network"
1824
networkclient "github.com/openshift/origin/pkg/network/generated/internalclientset"
@@ -42,12 +48,33 @@ type NetworkConfig struct {
4248
SDNProxy network.ProxyInterface
4349
}
4450

51+
// configureKubeConfigForClientCertRotation attempts to watch for client certificate rotation on the kubelet's cert
52+
// dir, if configured. This allows the network component to participate in client cert rotation when it is in the
53+
// same process (since it can't share a client with the Kubelet). This code path will be removed or altered when
54+
// the network process is split into a daemonset.
55+
func configureKubeConfigForClientCertRotation(options configapi.NodeConfig, kubeConfig *rest.Config) error {
56+
v, ok := options.KubeletArguments["cert-dir"]
57+
if !ok || len(v) == 0 {
58+
return nil
59+
}
60+
certDir := v[0]
61+
// equivalent to values in pkg/kubelet/certificate/kubelet.go
62+
store, err := certificate.NewFileStore("kubelet-client", certDir, certDir, kubeConfig.TLSClientConfig.CertFile, kubeConfig.TLSClientConfig.KeyFile)
63+
if err != nil {
64+
return err
65+
}
66+
return transport.RefreshCertificateAfterExpiry(utilwait.NeverStop, 10*time.Second, kubeConfig, store)
67+
}
68+
4569
// New creates a new network config object for running the networking components of the OpenShift node.
4670
func New(options configapi.NodeConfig, clusterDomain string, proxyConfig *componentconfig.KubeProxyConfiguration, enableProxy, enableDNS bool) (*NetworkConfig, error) {
4771
kubeConfig, err := configapi.GetKubeConfigOrInClusterConfig(options.MasterKubeConfig, options.MasterClientConnectionOverrides)
4872
if err != nil {
4973
return nil, err
5074
}
75+
if err := configureKubeConfigForClientCertRotation(options, kubeConfig); err != nil {
76+
utilruntime.HandleError(fmt.Errorf("Unable to enable client certificate rotation for network components: %v", err))
77+
}
5178
internalKubeClient, err := kclientsetinternal.NewForConfig(kubeConfig)
5279
if err != nil {
5380
return nil, err
Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
/*
2+
Copyright 2017 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
// Extracted from k8s.io/kubernetes/pkg/kubelet/certificate/transport.go, will be removed
18+
// when openshift-sdn and the network components move out of the Kubelet. Is intended ONLY
19+
// to provide certificate rollover until 3.8/3.9.
20+
package transport
21+
22+
import (
23+
"context"
24+
"crypto/tls"
25+
"fmt"
26+
"net"
27+
"net/http"
28+
"sync"
29+
"time"
30+
31+
"github.com/golang/glog"
32+
33+
utilnet "k8s.io/apimachinery/pkg/util/net"
34+
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
35+
"k8s.io/apimachinery/pkg/util/wait"
36+
restclient "k8s.io/client-go/rest"
37+
"k8s.io/kubernetes/pkg/kubelet/certificate"
38+
)
39+
40+
// RefreshCertificateAfterExpiry instruments a restconfig with a transport that checks
41+
// disk to reload expired certificates.
42+
//
43+
// The config must not already provide an explicit transport.
44+
//
45+
// The returned transport periodically checks the manager to determine if the
46+
// certificate has changed. If it has, the transport shuts down all existing client
47+
// connections, forcing the client to re-handshake with the server and use the
48+
// new certificate.
49+
//
50+
// stopCh should be used to indicate when the transport is unused and doesn't need
51+
// to continue checking the manager.
52+
func RefreshCertificateAfterExpiry(stopCh <-chan struct{}, period time.Duration, clientConfig *restclient.Config, store certificate.Store) error {
53+
if clientConfig.Transport != nil {
54+
return fmt.Errorf("there is already a transport configured")
55+
}
56+
tlsConfig, err := restclient.TLSConfigFor(clientConfig)
57+
if err != nil {
58+
return fmt.Errorf("unable to configure TLS for the rest client: %v", err)
59+
}
60+
if tlsConfig == nil {
61+
tlsConfig = &tls.Config{}
62+
}
63+
manager := &certificateManager{store: store, minimumRefresh: period}
64+
tlsConfig.Certificates = nil
65+
tlsConfig.GetClientCertificate = func(requestInfo *tls.CertificateRequestInfo) (*tls.Certificate, error) {
66+
cert := manager.Current()
67+
if cert == nil {
68+
return &tls.Certificate{Certificate: nil}, nil
69+
}
70+
return cert, nil
71+
}
72+
73+
// Custom dialer that will track all connections it creates.
74+
t := &connTracker{
75+
dialer: &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second},
76+
conns: make(map[*closableConn]struct{}),
77+
}
78+
79+
lastCert := manager.Current()
80+
go wait.Until(func() {
81+
curr := manager.Current()
82+
if curr == nil || lastCert == curr {
83+
// Cert hasn't been rotated.
84+
return
85+
}
86+
lastCert = curr
87+
88+
glog.Infof("certificate rotation detected, shutting down client connections to start using new credentials")
89+
// The cert has been rotated. Close all existing connections to force the client
90+
// to reperform its TLS handshake with new cert.
91+
//
92+
// See: https://github.com/kubernetes-incubator/bootkube/pull/663#issuecomment-318506493
93+
t.closeAllConns()
94+
}, period, stopCh)
95+
96+
clientConfig.Transport = utilnet.SetTransportDefaults(&http.Transport{
97+
Proxy: http.ProxyFromEnvironment,
98+
TLSHandshakeTimeout: 10 * time.Second,
99+
TLSClientConfig: tlsConfig,
100+
MaxIdleConnsPerHost: 25,
101+
DialContext: t.DialContext, // Use custom dialer.
102+
})
103+
104+
// Zero out all existing TLS options since our new transport enforces them.
105+
clientConfig.CertData = nil
106+
clientConfig.KeyData = nil
107+
clientConfig.CertFile = ""
108+
clientConfig.KeyFile = ""
109+
clientConfig.CAData = nil
110+
clientConfig.CAFile = ""
111+
clientConfig.Insecure = false
112+
return nil
113+
}
114+
115+
// certificateManager reloads the requested certificate from disk when requested.
116+
type certificateManager struct {
117+
store certificate.Store
118+
minimumRefresh time.Duration
119+
120+
lock sync.Mutex
121+
cert *tls.Certificate
122+
lastCheck time.Time
123+
}
124+
125+
// Current retrieves the latest certificate from disk if it exists, or nil if
126+
// no certificate could be found. The last successfully loaded certificate will be
127+
// returned.
128+
func (m *certificateManager) Current() *tls.Certificate {
129+
m.lock.Lock()
130+
defer m.lock.Unlock()
131+
132+
// check whether the cert has expired and whether we've waited long enough since our last
133+
// check to look again
134+
cert := m.cert
135+
if cert != nil {
136+
now := time.Now()
137+
if now.After(cert.Leaf.NotAfter) {
138+
if now.Sub(m.lastCheck) > m.minimumRefresh {
139+
glog.V(2).Infof("Current client certificate is expired, checking from disk")
140+
cert = nil
141+
m.lastCheck = now
142+
}
143+
}
144+
}
145+
146+
// load the cert from disk and parse the leaf cert
147+
if cert == nil {
148+
glog.V(2).Infof("Refreshing client certificate from store")
149+
c, err := m.store.Current()
150+
if err != nil {
151+
utilruntime.HandleError(fmt.Errorf("Unable to load client certificate key pair from disk: %v", err))
152+
return nil
153+
}
154+
m.cert = c
155+
}
156+
return m.cert
157+
}
158+
159+
// connTracker is a dialer that tracks all open connections it creates.
160+
type connTracker struct {
161+
dialer *net.Dialer
162+
163+
mu sync.Mutex
164+
conns map[*closableConn]struct{}
165+
}
166+
167+
// closeAllConns forcibly closes all tracked connections.
168+
func (c *connTracker) closeAllConns() {
169+
c.mu.Lock()
170+
conns := c.conns
171+
c.conns = make(map[*closableConn]struct{})
172+
c.mu.Unlock()
173+
174+
for conn := range conns {
175+
conn.Close()
176+
}
177+
}
178+
179+
func (c *connTracker) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
180+
conn, err := c.dialer.DialContext(ctx, network, address)
181+
if err != nil {
182+
return nil, err
183+
}
184+
185+
closable := &closableConn{Conn: conn}
186+
187+
// Start tracking the connection
188+
c.mu.Lock()
189+
c.conns[closable] = struct{}{}
190+
c.mu.Unlock()
191+
192+
// When the connection is closed, remove it from the map. This will
193+
// be no-op if the connection isn't in the map, e.g. if closeAllConns()
194+
// is called.
195+
closable.onClose = func() {
196+
c.mu.Lock()
197+
delete(c.conns, closable)
198+
c.mu.Unlock()
199+
}
200+
201+
return closable, nil
202+
}
203+
204+
type closableConn struct {
205+
onClose func()
206+
net.Conn
207+
}
208+
209+
func (c *closableConn) Close() error {
210+
go c.onClose()
211+
return c.Conn.Close()
212+
}

0 commit comments

Comments
 (0)