Skip to content

Commit 327ba9b

Browse files
committed
Add support for a new plugin to manage blueprint routes and configure the
manager appropriately. And some more changes as per review comments.
1 parent 336b00e commit 327ba9b

File tree

8 files changed

+442
-40
lines changed

8 files changed

+442
-40
lines changed

hack/lib/start.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -615,6 +615,13 @@ function os::start::router() {
615615
oc adm router --config="${ADMIN_KUBECONFIG}" --images="${USE_IMAGES}" --service-account=router
616616
fi
617617

618+
# Note that when the haproxy config manager is set based on router type,
619+
# the env entry may need to be always set or removed (if defaulted).
620+
if [[ -n "${ROUTER_HAPROXY_CONFIG_MANAGER:-}" ]]; then
621+
os::log::debug "Changing the router DC to enable the haproxy config manager"
622+
oc set env dc/router -c router ROUTER_HAPROXY_CONFIG_MANAGER=true
623+
fi
624+
618625
# Set the SYN eater to make router reloads more robust
619626
if [[ -n "${DROP_SYN_DURING_RESTART:-}" ]]; then
620627
# Rewrite the DC for the router to add the environment variable into the pod definition

pkg/cmd/infra/router/template.go

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ type TemplateRouter struct {
122122
}
123123

124124
type TemplateRouterConfigManager struct {
125-
ConfigManagerName string
125+
UseHAProxyConfigManager bool
126126
CommitInterval time.Duration
127127
BlueprintRouteNamespace string
128128
BlueprintRouteLabelSelector string
@@ -162,7 +162,7 @@ func (o *TemplateRouter) Bind(flag *pflag.FlagSet) {
162162
flag.StringVar(&o.Ciphers, "ciphers", util.Env("ROUTER_CIPHERS", ""), "Specifies the cipher suites to use. You can choose a predefined cipher set ('modern', 'intermediate', or 'old') or specify exact cipher suites by passing a : separated list.")
163163
flag.BoolVar(&o.StrictSNI, "strict-sni", isTrue(util.Env("ROUTER_STRICT_SNI", "")), "Use strict-sni bind processing (do not use default cert).")
164164
flag.StringVar(&o.MetricsType, "metrics-type", util.Env("ROUTER_METRICS_TYPE", ""), "Specifies the type of metrics to gather. Supports 'haproxy'.")
165-
flag.StringVar(&o.ConfigManagerName, "config-manager", util.Env("ROUTER_CONFIG_MANAGER", ""), "Specifies the manager to use for dynamically configuring changes with the underlying router. Supports 'haproxy-manager'.")
165+
flag.BoolVar(&o.UseHAProxyConfigManager, "haproxy-config-manager", isTrue(util.Env("ROUTER_HAPROXY_CONFIG_MANAGER", "")), "Use the the haproxy config manager (and dynamic configuration API) to configure route and endpoint changes. Reduces the number of haproxy reloads needed on configuration changes.")
166166
flag.DurationVar(&o.CommitInterval, "commit-interval", getIntervalFromEnv("COMMIT_INTERVAL", defaultCommitInterval), "Controls how often to commit (to the actual config) all the changes made using the router specific dynamic configuration manager.")
167167
flag.StringVar(&o.BlueprintRouteNamespace, "blueprint-route-namespace", util.Env("ROUTER_BLUEPRINT_ROUTE_NAMESPACE", ""), "Specifies the namespace which contains the routes that serve as blueprints for the dynamic configuration manager.")
168168
flag.StringVar(&o.BlueprintRouteLabelSelector, "blueprint-route-labels", util.Env("ROUTER_BLUEPRINT_ROUTE_LABELS", ""), "A label selector to apply to the routes in the blueprint route namespace. These selected routes will serve as blueprints for the dynamic dynamic configuration manager.")
@@ -438,7 +438,8 @@ func (o *TemplateRouterOptions) Run() error {
438438
}
439439

440440
var cfgManager templateplugin.ConfigManager
441-
if o.ConfigManagerName == "haproxy-manager" {
441+
var blueprintPlugin router.Plugin
442+
if o.UseHAProxyConfigManager {
442443
blueprintRoutes, err := o.blueprintRoutes(routeclient)
443444
if err != nil {
444445
return err
@@ -452,6 +453,9 @@ func (o *TemplateRouterOptions) Run() error {
452453
WildcardRoutesAllowed: o.AllowWildcardRoutes,
453454
}
454455
cfgManager = haproxyconfigmanager.NewHAProxyConfigManager(cmopts)
456+
if len(o.BlueprintRouteNamespace) > 0 {
457+
blueprintPlugin = haproxyconfigmanager.NewBlueprintPlugin(cfgManager)
458+
}
455459
}
456460

457461
pluginCfg := templateplugin.TemplatePluginConfig{
@@ -505,6 +509,17 @@ func (o *TemplateRouterOptions) Run() error {
505509
controller := factory.Create(plugin, false, o.EnableIngress)
506510
controller.Run()
507511

512+
if blueprintPlugin != nil {
513+
// f is like factory but filters the routes based on the
514+
// blueprint route namespace and label selector (if any).
515+
f := o.RouterSelection.NewFactory(routeclient, projectclient.Project().Projects(), kc)
516+
f.LabelSelector = o.BlueprintRouteLabelSelector
517+
f.Namespace = o.BlueprintRouteNamespace
518+
f.ResyncInterval = o.ResyncInterval
519+
c := f.Create(blueprintPlugin, false, false)
520+
c.Run()
521+
}
522+
508523
proc.StartReaper()
509524

510525
select {}

pkg/oc/admin/router/router.go

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -684,14 +684,11 @@ func RunCmdRouter(f *clientcmd.Factory, cmd *cobra.Command, out, errout io.Write
684684
env["ROUTER_CANONICAL_HOSTNAME"] = cfg.RouterCanonicalHostname
685685
}
686686
// automatically start the internal metrics agent if we are handling a known type
687-
if cfg.Type == "haproxy-router" {
688-
env["ROUTER_CONFIG_MANAGER"] = "haproxy-manager"
689-
if cfg.StatsPort != 0 {
690-
env["ROUTER_LISTEN_ADDR"] = fmt.Sprintf("0.0.0.0:%d", cfg.StatsPort)
691-
env["ROUTER_METRICS_TYPE"] = "haproxy"
692-
env["ROUTER_METRICS_TLS_CERT_FILE"] = "/etc/pki/tls/metrics/tls.crt"
693-
env["ROUTER_METRICS_TLS_KEY_FILE"] = "/etc/pki/tls/metrics/tls.key"
694-
}
687+
if cfg.Type == "haproxy-router" && cfg.StatsPort != 0 {
688+
env["ROUTER_LISTEN_ADDR"] = fmt.Sprintf("0.0.0.0:%d", cfg.StatsPort)
689+
env["ROUTER_METRICS_TYPE"] = "haproxy"
690+
env["ROUTER_METRICS_TLS_CERT_FILE"] = "/etc/pki/tls/metrics/tls.crt"
691+
env["ROUTER_METRICS_TLS_KEY_FILE"] = "/etc/pki/tls/metrics/tls.key"
695692
}
696693
env.Add(secretEnv)
697694
if len(defaultCert) > 0 {
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
package haproxy
2+
3+
import (
4+
"k8s.io/apimachinery/pkg/util/sets"
5+
"k8s.io/apimachinery/pkg/watch"
6+
kapi "k8s.io/kubernetes/pkg/apis/core"
7+
8+
routeapi "github.com/openshift/origin/pkg/route/apis/route"
9+
templaterouter "github.com/openshift/origin/pkg/router/template"
10+
)
11+
12+
// BlueprintPlugin implements the router.Plugin interface to process routes
13+
// from the blueprint namespace for the associated config manager.
14+
type BlueprintPlugin struct {
15+
manager templaterouter.ConfigManager
16+
}
17+
18+
// NewBlueprintPlugin returns a new blueprint routes plugin.
19+
func NewBlueprintPlugin(cm templaterouter.ConfigManager) *BlueprintPlugin {
20+
return &BlueprintPlugin{manager: cm}
21+
}
22+
23+
// HandleRoute processes watch events on blueprint routes.
24+
func (p *BlueprintPlugin) HandleRoute(eventType watch.EventType, route *routeapi.Route) error {
25+
switch eventType {
26+
case watch.Added, watch.Modified:
27+
p.manager.AddBlueprint(route)
28+
case watch.Deleted:
29+
p.manager.RemoveBlueprint(route)
30+
}
31+
32+
return nil
33+
}
34+
35+
// HandleNode processes watch events on the Node resource.
36+
func (p *BlueprintPlugin) HandleNode(eventType watch.EventType, node *kapi.Node) error {
37+
return nil
38+
}
39+
40+
// HandleEndpoints processes watch events on the Endpoints resource.
41+
func (p *BlueprintPlugin) HandleEndpoints(eventType watch.EventType, endpoints *kapi.Endpoints) error {
42+
return nil
43+
}
44+
45+
// HandleNamespaces processes watch events on namespaces.
46+
func (p *BlueprintPlugin) HandleNamespaces(namespaces sets.String) error {
47+
return nil
48+
}
49+
50+
// Commit commits the changes made to a watched resource.
51+
func (p *BlueprintPlugin) Commit() error {
52+
// Nothing to do as the config manager does an automatic commit when
53+
// any blueprint routes change.
54+
return nil
55+
}
Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
package haproxy
2+
3+
import (
4+
"fmt"
5+
"testing"
6+
"time"
7+
8+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
9+
"k8s.io/apimachinery/pkg/util/sets"
10+
"k8s.io/apimachinery/pkg/watch"
11+
kapi "k8s.io/kubernetes/pkg/apis/core"
12+
13+
routeapi "github.com/openshift/origin/pkg/route/apis/route"
14+
templaterouter "github.com/openshift/origin/pkg/router/template"
15+
)
16+
17+
type fakeConfigManager struct {
18+
blueprints map[string]*routeapi.Route
19+
}
20+
21+
func newFakeConfigManager() *fakeConfigManager {
22+
return &fakeConfigManager{
23+
blueprints: make(map[string]*routeapi.Route),
24+
}
25+
}
26+
27+
func (cm *fakeConfigManager) Initialize(router templaterouter.RouterInterface, certPath string) {
28+
}
29+
30+
func (cm *fakeConfigManager) AddBlueprint(route *routeapi.Route) {
31+
cm.blueprints[routeKey(route)] = route
32+
}
33+
34+
func (cm *fakeConfigManager) RemoveBlueprint(route *routeapi.Route) {
35+
delete(cm.blueprints, routeKey(route))
36+
}
37+
38+
func (cm *fakeConfigManager) FindBlueprint(id string) (*routeapi.Route, bool) {
39+
route, ok := cm.blueprints[id]
40+
return route, ok
41+
}
42+
43+
func (cm *fakeConfigManager) Register(id string, route *routeapi.Route) {
44+
}
45+
46+
func (cm *fakeConfigManager) AddRoute(id string, route *routeapi.Route) error {
47+
return nil
48+
}
49+
50+
func (cm *fakeConfigManager) RemoveRoute(id string, route *routeapi.Route) error {
51+
return nil
52+
}
53+
54+
func (cm *fakeConfigManager) ReplaceRouteEndpoints(id string, oldEndpoints, newEndpoints []templaterouter.Endpoint, weight int32) error {
55+
return nil
56+
}
57+
58+
func (cm *fakeConfigManager) RemoveRouteEndpoints(id string, endpoints []templaterouter.Endpoint) error {
59+
return nil
60+
}
61+
62+
func (cm *fakeConfigManager) Notify(event templaterouter.RouterEventType) {
63+
}
64+
65+
func (cm *fakeConfigManager) ServerTemplateName(id string) string {
66+
return "fakeConfigManager"
67+
}
68+
69+
func (cm *fakeConfigManager) ServerTemplateSize(id string) string {
70+
return "1"
71+
}
72+
73+
func (cm *fakeConfigManager) GenerateDynamicServerNames(id string) []string {
74+
return []string{}
75+
}
76+
77+
func routeKey(route *routeapi.Route) string {
78+
return fmt.Sprintf("%s:%s", route.Name, route.Namespace)
79+
}
80+
81+
// TestHandleRoute test route watch events
82+
func TestHandleRoute(t *testing.T) {
83+
original := metav1.Time{Time: time.Now()}
84+
85+
route := &routeapi.Route{
86+
ObjectMeta: metav1.ObjectMeta{
87+
CreationTimestamp: original,
88+
Namespace: "bp",
89+
Name: "chevron",
90+
},
91+
Spec: routeapi.RouteSpec{
92+
Host: "www.blueprints.org",
93+
To: routeapi.RouteTargetReference{
94+
Name: "TestService",
95+
Weight: new(int32),
96+
},
97+
},
98+
}
99+
100+
cm := newFakeConfigManager()
101+
plugin := NewBlueprintPlugin(cm)
102+
plugin.HandleRoute(watch.Added, route)
103+
104+
id := routeKey(route)
105+
if _, ok := cm.FindBlueprint(id); !ok {
106+
t.Errorf("TestHandleRoute was unable to find a blueprint %s after HandleRoute was called", id)
107+
}
108+
109+
// update a blueprint with a newer time and host
110+
v2route := route.DeepCopy()
111+
v2route.CreationTimestamp = metav1.Time{Time: original.Add(time.Hour)}
112+
v2route.Spec.Host = "updated.blueprint.org"
113+
if err := plugin.HandleRoute(watch.Added, v2route); err != nil {
114+
t.Errorf("TestHandleRoute unexpected error after blueprint update: %v", err)
115+
}
116+
117+
blueprints := []*routeapi.Route{v2route, route}
118+
for _, r := range blueprints {
119+
// delete the blueprint and check that it doesn't exist.
120+
if err := plugin.HandleRoute(watch.Deleted, v2route); err != nil {
121+
t.Errorf("TestHandleRoute unexpected error after blueprint delete: %v", err)
122+
}
123+
124+
routeId := routeKey(r)
125+
if _, ok := cm.FindBlueprint(routeId); ok {
126+
t.Errorf("TestHandleRoute found a blueprint %s after it was deleted", routeId)
127+
}
128+
}
129+
}
130+
131+
func TestHandleNode(t *testing.T) {
132+
node := &kapi.Node{
133+
ObjectMeta: metav1.ObjectMeta{
134+
Labels: map[string]string{"design": "blueprint"},
135+
},
136+
}
137+
138+
cm := newFakeConfigManager()
139+
plugin := NewBlueprintPlugin(cm)
140+
141+
if err := plugin.HandleNode(watch.Added, node); err != nil {
142+
t.Errorf("TestHandleNode unexpected error after node add: %v", err)
143+
}
144+
145+
if err := plugin.HandleNode(watch.Modified, node); err != nil {
146+
t.Errorf("TestHandleNode unexpected error after node modify: %v", err)
147+
}
148+
149+
if err := plugin.HandleNode(watch.Deleted, node); err != nil {
150+
t.Errorf("TestHandleNode unexpected error after node delete: %v", err)
151+
}
152+
}
153+
154+
func TestHandleEndpoints(t *testing.T) {
155+
endpoints := &kapi.Endpoints{
156+
ObjectMeta: metav1.ObjectMeta{
157+
Namespace: "bpe",
158+
Name: "shell",
159+
},
160+
Subsets: []kapi.EndpointSubset{{
161+
Addresses: []kapi.EndpointAddress{{IP: "1.1.1.1"}},
162+
Ports: []kapi.EndpointPort{{Port: 9876}},
163+
}},
164+
}
165+
166+
v2Endpoints := &kapi.Endpoints{
167+
ObjectMeta: metav1.ObjectMeta{
168+
Namespace: "bpe",
169+
Name: "shell",
170+
},
171+
Subsets: []kapi.EndpointSubset{{
172+
Addresses: []kapi.EndpointAddress{{IP: "1.1.1.1"}, {IP: "2.2.2.2"}},
173+
Ports: []kapi.EndpointPort{{Port: 9876}, {Port: 8888}},
174+
}},
175+
}
176+
177+
cm := newFakeConfigManager()
178+
plugin := NewBlueprintPlugin(cm)
179+
180+
if err := plugin.HandleEndpoints(watch.Added, endpoints); err != nil {
181+
t.Errorf("TestHandleEndpoints unexpected error after endpoints add: %v", err)
182+
}
183+
184+
if err := plugin.HandleEndpoints(watch.Modified, v2Endpoints); err != nil {
185+
t.Errorf("TestHandleEndpoints unexpected error after endpoints modify: %v", err)
186+
}
187+
188+
if err := plugin.HandleEndpoints(watch.Deleted, v2Endpoints); err != nil {
189+
t.Errorf("TestHandleEndpoints unexpected error after endpoints delete: %v", err)
190+
}
191+
}
192+
193+
func TestHandleNamespaces(t *testing.T) {
194+
cm := newFakeConfigManager()
195+
plugin := NewBlueprintPlugin(cm)
196+
197+
if err := plugin.HandleNamespaces(sets.String{}); err != nil {
198+
t.Errorf("TestHandleNamespaces unexpected error after empty set: %v", err)
199+
}
200+
201+
if err := plugin.HandleNamespaces(sets.NewString("76")); err != nil {
202+
t.Errorf("TestHandleNamespaces unexpected error after set: %v", err)
203+
}
204+
205+
if err := plugin.HandleNamespaces(sets.NewString("76", "711")); err != nil {
206+
t.Errorf("TestHandleNamespaces unexpected error after set multiple: %v", err)
207+
}
208+
209+
if err := plugin.HandleNamespaces(sets.NewString("arco")); err != nil {
210+
t.Errorf("TestHandleNamespaces unexpected error after reset: %v", err)
211+
}
212+
}

0 commit comments

Comments
 (0)