ab testing

Author: Rajat Chopra

api/swagger-spec/oapi-v1.json

@@ -22799,9 +22799,15 @@
       "description": "Path that the router watches for, to route traffic for to the service. Optional"
      },
      "to": {
-      "$ref": "v1.ObjectReference",
+      "$ref": "v1.RouteTargetReference",
       "description": "To is an object the route points to. Only the Service kind is allowed, and it will be defaulted to Service."
      },
+     "additionalTos": {
+      "type": "array",
+      "items": {
+       "$ref": "v1.RouteTargetReference"
+      }
+     },
      "port": {
       "$ref": "v1.RoutePort",
       "description": "If specified, the port to be used by the router. Most routers will use all endpoints exposed by the service by default - set this value to instruct routers which port to use."
@@ -22812,6 +22818,26 @@
      }
     }
    },
+   "v1.RouteTargetReference": {
+    "id": "v1.RouteTargetReference",
+    "required": [
+     "kind",
+     "name",
+     "weight"
+    ],
+    "properties": {
+     "kind": {
+      "type": "string"
+     },
+     "name": {
+      "type": "string"
+     },
+     "weight": {
+      "type": "integer",
+      "format": "int32"
+     }
+    }
+   },
    "v1.RoutePort": {
     "id": "v1.RoutePort",
     "description": "RoutePort defines a port mapping from a router to an endpoint in the service endpoints.",

images/router/haproxy/conf/haproxy-config.template

@@ -206,18 +206,25 @@ backend openshift_default
             where to send the traffic but should run the be in tcp mode
         3. if the config is terminated at the
 */}}
-{{ range $id, $serviceUnit := .State }}
-        {{ range $cfgIdx, $cfg := $serviceUnit.ServiceAliasConfigs }}
-            {{ if or (eq $cfg.TLSTermination "") (eq $cfg.TLSTermination "edge") }}
-                {{ if (eq $cfg.TLSTermination "") }}
+{{ range $cfgIdx, $cfg := .State }}
+  {{ if or (eq $cfg.TLSTermination "") (eq $cfg.TLSTermination "edge") }}
+    {{ if (eq $cfg.TLSTermination "") }}
+# Plain http backend
 backend be_http_{{$cfgIdx}}
-                {{ else }}
+    {{ else }}
+# Plain http backend but request is TLS, terminated at edge
 backend be_edge_http_{{$cfgIdx}}
-                {{ end }}
+    {{ end }}
   mode http
   option redispatch
   option forwardfor
+    {{ with $balanceAlgo := index $cfg.Annotations "router.openshift.io/haproxy.balance" }}
+      {{ with $matchValue := (matchValues $balanceAlgo "roundrobin" "leastconn" ) }}
+  balance {{ $balanceAlgo }}
+      {{ end }}
+    {{ else }}
   balance leastconn
+    {{ end }}
   timeout check 5000ms
   http-request set-header X-Forwarded-Host %[req.hdr(host)]
   http-request set-header X-Forwarded-Port %[dst_port]
@@ -226,41 +233,53 @@ backend be_edge_http_{{$cfgIdx}}
   {{ if (eq $cfg.TLSTermination "") }}
     cookie {{$cfg.RoutingKeyName}} insert indirect nocache httponly
   {{ else }}
-    cookie {{$cfg.RoutingKeyName}} insert indirect nocache httponly secure
+  cookie {{$cfg.RoutingKeyName}} insert indirect nocache httponly secure
   {{ end }}
   http-request set-header Forwarded for=%[src];host=%[req.hdr(host)];proto=%[req.hdr(X-Forwarded-Proto)]
-                {{ range $idx, $endpoint := endpointsForAlias $cfg $serviceUnit }}
-  server {{$endpoint.IdHash}} {{$endpoint.IP}}:{{$endpoint.Port}} check inter 5000ms cookie {{$endpoint.IdHash}}
-                {{ end }}
-            {{ end }}
-
-            {{ if eq $cfg.TLSTermination "passthrough" }}
+    {{ range $serviceUnitName, $weight := $cfg.ServiceUnitNames }}
+      {{ with $serviceUnit := index $.ServiceUnits $serviceUnitName }}
+          {{ range $idx, $endpoint := endpointsForAlias $cfg $serviceUnit }}
+  server {{$endpoint.IdHash}} {{$endpoint.IP}}:{{$endpoint.Port}} check inter 5000ms cookie {{$endpoint.IdHash}} weight {{ $weight }}
+          {{ end }}
+      {{ end }}
+    {{ end }}{{/* end iterate over services */}}
+  {{ end }}{{/* end if tls==edge/none */}}
+
+# Secure backend, pass through
+  {{ if eq $cfg.TLSTermination "passthrough" }}
 backend be_tcp_{{$cfgIdx}}
 {{ if ne (env "ROUTER_SYSLOG_ADDRESS" "") ""}}
   option tcplog
 {{ end }}
   balance {{ env "ROUTER_TCP_BALANCE_SCHEME" "source" }}
   hash-type consistent
   timeout check 5000ms
-                {{ range $idx, $endpoint := endpointsForAlias $cfg $serviceUnit }}
+    {{ range $svcUnitIdx, $serviceUnitName := $cfg.ServiceUnitNames }}
+      {{ with $serviceUnit := index $.ServiceUnits $serviceUnitName }}
+        {{ range $idx, $endpoint := endpointsForAlias $cfg $serviceUnit }}
   server {{$endpoint.ID}} {{$endpoint.IP}}:{{$endpoint.Port}} check inter 5000ms
-                {{ end }}
-            {{ end }}
+        {{ end }}
+      {{ end }}
+    {{ end }}{{/* end iterate over services*/}}
+  {{ end }}{{/*end tls==passthrough*/}}
 
-            {{ if eq $cfg.TLSTermination "reencrypt" }}
+# Secure backend which requires re-encryption
+  {{ if eq $cfg.TLSTermination "reencrypt" }}
 backend be_secure_{{$cfgIdx}}
   mode http
   option redispatch
   balance leastconn
   timeout check 5000ms
   cookie {{$cfg.RoutingKeyName}} insert indirect nocache httponly secure
-                {{ range $idx, $endpoint := endpointsForAlias $cfg $serviceUnit }}
+    {{ range $svcUnitIdx, $serviceUnitName := $cfg.ServiceUnitNames }}
+      {{ with $serviceUnit := index $.ServiceUnits $serviceUnitName }}
+        {{ range $idx, $endpoint := endpointsForAlias $cfg $serviceUnit }}
   server {{$endpoint.IdHash}} {{$endpoint.IP}}:{{$endpoint.Port}} ssl check inter 5000ms verify required ca-file {{ $workingDir }}/cacerts/{{$cfgIdx}}.pem cookie {{$endpoint.IdHash}}
-                {{ end }}
-            {{ end  }}
-        {{ end  }}{{/* $serviceUnit.ServiceAliasConfigs*/}}
-{{ end }}{{/* $serviceUnit */}}
-
+        {{ end }}
+      {{ end }}
+    {{ end }}
+  {{ end }}{{/* end tls==reencrypt */}}
+{{ end }}{{/* end loop over routes */}}
 {{ end }}{{/* end haproxy config template */}}
 
 {{/*--------------------------------- END OF HAPROXY CONFIG, BELOW ARE MAPPING FILES ------------------------*/}}
@@ -269,27 +288,23 @@ backend be_secure_{{$cfgIdx}}
                         by attaching a prefix (be_http_) by use_backend statements if acls are matched.
 */}}
 {{ define "/var/lib/haproxy/conf/os_http_be.map" }}
-{{   range $id, $serviceUnit := .State }}
-{{     range $idx, $cfg := $serviceUnit.ServiceAliasConfigs }}
+{{     range $idx, $cfg := .State }}
 {{       if and (ne $cfg.Host "") (eq $cfg.TLSTermination "")}}
 {{$cfg.Host}}{{$cfg.Path}} {{$idx}}
 {{       end }}
 {{     end }}
-{{   end }}
 {{ end }}{{/* end http host map template */}}
 
 {{/*
     os_edge_http_be.map: same as os_http_be.map but allows us to separate tls from non-tls routes to ensure we don't expose
                             a tls only route on the unsecure port
 */}}
 {{ define "/var/lib/haproxy/conf/os_edge_http_be.map" }}
-{{   range $id, $serviceUnit := .State }}
-{{     range $idx, $cfg := $serviceUnit.ServiceAliasConfigs }}
+{{     range $idx, $cfg := .State }}
 {{       if and (ne $cfg.Host "") (eq $cfg.TLSTermination "edge")}}
 {{$cfg.Host}}{{$cfg.Path}} {{$idx}}
 {{       end }}
 {{     end }}
-{{   end }}
 {{ end }}{{/* end edge http host map template */}}
 
 {{/*
@@ -298,13 +313,11 @@ backend be_secure_{{$cfgIdx}}
     (http) if acls match for routes with insecure option set to expose.
 */}}
 {{ define "/var/lib/haproxy/conf/os_edge_http_expose.map" }}
-{{   range $id, $serviceUnit := .State }}
-{{     range $idx, $cfg := $serviceUnit.ServiceAliasConfigs }}
+{{     range $idx, $cfg := .State }}
 {{       if and (ne $cfg.Host "") (and (eq $cfg.TLSTermination "edge") (eq $cfg.InsecureEdgeTerminationPolicy "Allow"))}}
 {{$cfg.Host}}{{$cfg.Path}} {{$idx}}
 {{       end }}
 {{     end }}
-{{   end }}
 {{ end }}{{/* end edge insecure expose http host map template */}}
 
 {{/*
@@ -313,13 +326,11 @@ backend be_secure_{{$cfgIdx}}
     if acls match for routes that have the insecure option set to redirect.
 */}}
 {{ define "/var/lib/haproxy/conf/os_edge_http_redirect.map" }}
-{{   range $id, $serviceUnit := .State }}
-{{     range $idx, $cfg := $serviceUnit.ServiceAliasConfigs }}
+{{     range $idx, $cfg := .State }}
 {{       if and (ne $cfg.Host "") (and (eq $cfg.TLSTermination "edge") (eq $cfg.InsecureEdgeTerminationPolicy "Redirect"))}}
 {{$cfg.Host}}{{$cfg.Path}} {{$idx}}
 {{       end }}
 {{     end }}
-{{   end }}
 {{ end }}{{/* end edge insecure redirect http host map template */}}
 
 
@@ -328,27 +339,23 @@ backend be_secure_{{$cfgIdx}}
                         by attaching a prefix (be_tcp_ or be_secure_) by use_backend statements if acls are matched.
 */}}
 {{ define "/var/lib/haproxy/conf/os_tcp_be.map" }}
-{{   range $id, $serviceUnit := .State }}
-{{     range $idx, $cfg := $serviceUnit.ServiceAliasConfigs }}
+{{     range $idx, $cfg := .State }}
 {{       if and (eq $cfg.Path "") (and (ne $cfg.Host "") (or (eq $cfg.TLSTermination "passthrough") (eq $cfg.TLSTermination "reencrypt"))) }}
 {{$cfg.Host}} {{$idx}}
 {{       end }}
 {{     end }}
-{{   end }}
 {{ end }}{{/* end tcp host map template */}}
 
 {{/*
     os_sni_passthrough.map: contains a mapping of routes that expect to have an sni header and should be passed
     					through to the host_be.  Driven by the termination type of the ServiceAliasConfigs
 */}}
 {{ define "/var/lib/haproxy/conf/os_sni_passthrough.map" }}
-{{   range $id, $serviceUnit := .State }}
-{{     range $idx, $cfg := $serviceUnit.ServiceAliasConfigs }}
+{{     range $idx, $cfg := .State }}
 {{       if and (eq $cfg.Path "") (eq $cfg.TLSTermination "passthrough") }}
 {{$cfg.Host}} 1
 {{       end }}
 {{     end }}
-{{   end }}
 {{ end }}{{/* end sni passthrough map template */}}
 
 
@@ -357,11 +364,9 @@ backend be_secure_{{$cfgIdx}}
                     that does specific checks that avoid mitm attacks: http://cbonte.github.io/haproxy-dconv/configuration-1.5.html#5.2-ssl
 */}}
 {{ define "/var/lib/haproxy/conf/os_reencrypt.map" }}
-{{   range $id, $serviceUnit := .State }}
-{{     range $idx, $cfg := $serviceUnit.ServiceAliasConfigs }}
+{{     range $idx, $cfg := .State }}
 {{       if and (ne $cfg.Host "") (eq $cfg.TLSTermination "reencrypt") }}
 {{$cfg.Host}}{{$cfg.Path}} {{$idx}}
 {{       end }}
 {{     end }}
-{{   end }}
 {{ end }}{{/* end reencrypt passthrough map template */}}

pkg/cmd/cli/cmd/create_route.go

@@ -373,7 +373,7 @@ func unsecuredRoute(kc *kclient.Client, namespace, routeName, serviceName, portS
 				Name: routeName,
 			},
 			Spec: api.RouteSpec{
-				To: kapi.ObjectReference{
+				To: api.RouteTargetReference{
 					Name: serviceName,
 				},
 				Port: resolveRoutePort(portString),
@@ -392,7 +392,7 @@ func unsecuredRoute(kc *kclient.Client, namespace, routeName, serviceName, portS
 			Labels: svc.Labels,
 		},
 		Spec: api.RouteSpec{
-			To: kapi.ObjectReference{
+			To: api.RouteTargetReference{
 				Name: serviceName,
 			},
 		},

pkg/generate/app/pipeline.go

@@ -382,7 +382,7 @@ func AddRoutes(objects Objects) Objects {
 					Labels: t.Labels,
 				},
 				Spec: route.RouteSpec{
-					To: kapi.ObjectReference{
+					To: route.RouteTargetReference{
 						Name: t.Name,
 					},
 				},

pkg/route/allocation/simple/plugin.go

@@ -38,7 +38,7 @@ func NewSimpleAllocationPlugin(suffix string) (*SimpleAllocationPlugin, error) {
 // the "global" router shard.
 // TODO: replace with per router allocation
 func (p *SimpleAllocationPlugin) Allocate(route *routeapi.Route) (*routeapi.RouterShard, error) {
-	glog.V(4).Infof("Allocating global shard *.%s to Route: %s", p.DNSSuffix, route.Spec.To.Name)
+	glog.V(4).Infof("Allocating global shard *.%s to Route: %s", p.DNSSuffix, route.Name)
 
 	return &routeapi.RouterShard{ShardName: "global", DNSSuffix: p.DNSSuffix}, nil
 }

pkg/route/api/deep_copy_generated.go

@@ -20,6 +20,7 @@ func init() {
 		DeepCopy_api_RoutePort,
 		DeepCopy_api_RouteSpec,
 		DeepCopy_api_RouteStatus,
+		DeepCopy_api_RouteTargetReference,
 		DeepCopy_api_RouterShard,
 		DeepCopy_api_TLSConfig,
 	); err != nil {
@@ -109,9 +110,20 @@ func DeepCopy_api_RoutePort(in RoutePort, out *RoutePort, c *conversion.Cloner)
 func DeepCopy_api_RouteSpec(in RouteSpec, out *RouteSpec, c *conversion.Cloner) error {
 	out.Host = in.Host
 	out.Path = in.Path
-	if err := api.DeepCopy_api_ObjectReference(in.To, &out.To, c); err != nil {
+	if err := DeepCopy_api_RouteTargetReference(in.To, &out.To, c); err != nil {
 		return err
 	}
+	if in.AdditionalTos != nil {
+		in, out := in.AdditionalTos, &out.AdditionalTos
+		*out = make([]RouteTargetReference, len(in))
+		for i := range in {
+			if err := DeepCopy_api_RouteTargetReference(in[i], &(*out)[i], c); err != nil {
+				return err
+			}
+		}
+	} else {
+		out.AdditionalTos = nil
+	}
 	if in.Port != nil {
 		in, out := in.Port, &out.Port
 		*out = new(RoutePort)
@@ -148,6 +160,19 @@ func DeepCopy_api_RouteStatus(in RouteStatus, out *RouteStatus, c *conversion.Cl
 	return nil
 }
 
+func DeepCopy_api_RouteTargetReference(in RouteTargetReference, out *RouteTargetReference, c *conversion.Cloner) error {
+	out.Kind = in.Kind
+	out.Name = in.Name
+	if in.Weight != nil {
+		in, out := in.Weight, &out.Weight
+		*out = new(int32)
+		**out = *in
+	} else {
+		out.Weight = nil
+	}
+	return nil
+}
+
 func DeepCopy_api_RouterShard(in RouterShard, out *RouterShard, c *conversion.Cloner) error {
 	out.ShardName = in.ShardName
 	out.DNSSuffix = in.DNSSuffix

pkg/route/api/types.go

@@ -25,9 +25,13 @@ type RouteSpec struct {
 	// Path that the router watches for, to route traffic for to the service. Optional
 	Path string
 
-	// An object the route points to. Only the Service kind is allowed, and it will
+	// Objects that the route points to. Only the Service kind is allowed, and it will
 	// be defaulted to Service.
-	To kapi.ObjectReference
+	To RouteTargetReference
+
+	// Additional objects that the route may want to point to. Use the 'weight' field to
+	// determine which ones of the several get more emphasis
+	AdditionalTos []RouteTargetReference
 
 	// If specified, the port to be used by the router. Most routers will use all
 	// endpoints exposed by the service by default - set this value to instruct routers
@@ -38,6 +42,14 @@ type RouteSpec struct {
 	TLS *TLSConfig
 }
 
+// RouteTargetReference specifies the target that resolve into endpoints. Only the 'Service'
+// kind is allowed. Use 'weight' field to emphasize one over others.
+type RouteTargetReference struct {
+	Kind   string
+	Name   string
+	Weight *int32
+}
+
 // RoutePort defines a port mapping from a router to an endpoint in the service endpoints.
 type RoutePort struct {
 	// The target port on pods selected by the service this route points to.

pkg/route/api/v1/conversion.go

@@ -10,8 +10,16 @@ import (
 func addConversionFuncs(scheme *runtime.Scheme) {
 	err := scheme.AddDefaultingFuncs(
 		func(obj *RouteSpec) {
-			if len(obj.To.Kind) == 0 {
-				obj.To.Kind = "Service"
+			if obj.AdditionalTos == nil {
+				obj.AdditionalTos = make([]RouteTargetReference, 0)
+			}
+		},
+		func(obj *RouteTargetReference) {
+			if len(obj.Kind) == 0 {
+				obj.Kind = "Service"
+			}
+			if obj.Weight == nil {
+				*obj.Weight = 100
 			}
 		},
 		func(obj *TLSConfig) {