From 417e073f09096f54974022b5381606a38bfa68b1 Mon Sep 17 00:00:00 2001 From: Jacob Tanenbaum Date: Fri, 11 May 2018 11:33:46 -0400 Subject: [PATCH 1/2] UPSTREAM: 63716: Add InstallPathHandler which allows for more then one path to be associated with health checking. Currently it is only possible to have one group of checks which must all pass for the handler to report success. Allowing multiple paths for these checks allows use of the same machinery for other kinds of checks, i.e. readiness. This upstream change allows for the differentiation of health and readiness checks --- .../apiserver/pkg/server/healthz/healthz.go | 13 ++++++-- .../pkg/server/healthz/healthz_test.go | 32 +++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go b/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go index bd8bfe7b53d9..224f1eda2c44 100644 --- a/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go +++ b/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go @@ -66,6 +66,15 @@ func NamedCheck(name string, check func(r *http.Request) error) HealthzChecker { // exactly one call to InstallHandler. Calling InstallHandler more // than once for the same mux will result in a panic. func InstallHandler(mux mux, checks ...HealthzChecker) { + InstallPathHandler(mux, "/healthz", checks...) +} + +// InstallPathHandler registers handlers for health checking on +// a specific path to mux. *All handlers* for the path must be +// specified in exactly one call to InstallPathHandler. Calling +// InstallPathHandler more than once for the same path and mux will +// result in a panic. +func InstallPathHandler(mux mux, path string, checks ...HealthzChecker) { if len(checks) == 0 { glog.V(5).Info("No default health checks specified. Installing the ping handler.") checks = []HealthzChecker{PingHealthz} @@ -73,9 +82,9 @@ func InstallHandler(mux mux, checks ...HealthzChecker) { glog.V(5).Info("Installing healthz checkers:", strings.Join(checkerNames(checks...), ", ")) - mux.Handle("/healthz", handleRootHealthz(checks...)) + mux.Handle(path, handleRootHealthz(checks...)) for _, check := range checks { - mux.Handle(fmt.Sprintf("/healthz/%v", check.Name()), adaptCheckToHandler(check.Check)) + mux.Handle(fmt.Sprintf("%s/%v", path, check.Name()), adaptCheckToHandler(check.Check)) } } diff --git a/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz_test.go b/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz_test.go index 99aae5a9eabd..a1f761b786b0 100644 --- a/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz_test.go +++ b/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz_test.go @@ -42,6 +42,38 @@ func TestInstallHandler(t *testing.T) { } } +func TestInstallPathHandler(t *testing.T) { + mux := http.NewServeMux() + InstallPathHandler(mux, "/healthz/test") + InstallPathHandler(mux, "/healthz/ready") + req, err := http.NewRequest("GET", "http://example.com/healthz/test", nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Errorf("expected %v, got %v", http.StatusOK, w.Code) + } + if w.Body.String() != "ok" { + t.Errorf("expected %v, got %v", "ok", w.Body.String()) + } + + req, err = http.NewRequest("GET", "http://example.com/healthz/ready", nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + w = httptest.NewRecorder() + mux.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Errorf("expected %v, got %v", http.StatusOK, w.Code) + } + if w.Body.String() != "ok" { + t.Errorf("expected %v, got %v", "ok", w.Body.String()) + } + +} + func TestMulitipleChecks(t *testing.T) { tests := []struct { path string From 978d2bc3de43445e4809193016ee7f658ca1348a Mon Sep 17 00:00:00 2001 From: Jacob Tanenbaum Date: Fri, 11 May 2018 11:34:20 -0400 Subject: [PATCH 2/2] Differentiate liveness and readiness probes for router Add a backend to the router controller "/livez" that always returns true. This differentiates the liveness and readiness probes so that a router can be alive and not ready. Bug 1550007 --- pkg/cmd/infra/router/template.go | 15 ++++++++++++--- pkg/oc/admin/router/router.go | 19 +++++++++---------- pkg/router/metrics/health.go | 23 +++++++++++++++++++++++ pkg/router/metrics/metrics.go | 6 ++++-- 4 files changed, 48 insertions(+), 15 deletions(-) diff --git a/pkg/cmd/infra/router/template.go b/pkg/cmd/infra/router/template.go index 1e81c0a4c2d7..10956fd2fce5 100644 --- a/pkg/cmd/infra/router/template.go +++ b/pkg/cmd/infra/router/template.go @@ -243,6 +243,7 @@ func (o *TemplateRouterOptions) Validate() error { // Run launches a template router using the provided options. It never exits. func (o *TemplateRouterOptions) Run() error { glog.Infof("Starting template router (%s)", version.Get()) + var ptrTemplatePlugin *templateplugin.TemplatePlugin var reloadCallbacks []func() @@ -310,9 +311,15 @@ func (o *TemplateRouterOptions) Run() error { if err != nil { return fmt.Errorf("ROUTER_METRICS_READY_HTTP_URL must be a valid URL or empty: %v", err) } - check := metrics.HTTPBackendAvailable(u) + checkBackend := metrics.HTTPBackendAvailable(u) if isTrue(util.Env("ROUTER_USE_PROXY_PROTOCOL", "")) { - check = metrics.ProxyProtocolHTTPBackendAvailable(u) + checkBackend = metrics.ProxyProtocolHTTPBackendAvailable(u) + } + checkSync := metrics.HasSynced(&ptrTemplatePlugin) + checkController := metrics.ControllerLive() + liveChecks := []healthz.HealthzChecker{checkController} + if !(isTrue(util.Env("ROUTER_BIND_PORTS_BEFORE_SYNC", ""))) { + liveChecks = append(liveChecks, checkBackend) } kubeconfig := o.Config.KubeConfig() @@ -353,7 +360,8 @@ func (o *TemplateRouterOptions) Run() error { Resource: "routers", Name: o.RouterName, }, - Checks: []healthz.HealthzChecker{check}, + LiveChecks: liveChecks, + ReadyChecks: []healthz.HealthzChecker{checkBackend, checkSync}, } if certFile := util.Env("ROUTER_METRICS_TLS_CERT_FILE", ""); len(certFile) > 0 { certificate, err := tls.LoadX509KeyPair(certFile, util.Env("ROUTER_METRICS_TLS_KEY_FILE", "")) @@ -411,6 +419,7 @@ func (o *TemplateRouterOptions) Run() error { if err != nil { return err } + ptrTemplatePlugin = templatePlugin factory := o.RouterSelection.NewFactory(routeclient, projectclient.Project().Projects(), kc) factory.RouteModifierFn = o.RouteUpdate diff --git a/pkg/oc/admin/router/router.go b/pkg/oc/admin/router/router.go index 47241e6e3fe2..704e140f966f 100644 --- a/pkg/oc/admin/router/router.go +++ b/pkg/oc/admin/router/router.go @@ -243,9 +243,8 @@ const ( // Default port numbers to expose and bind/listen on. defaultPorts = "80:80,443:443" - // Default stats and healthz port. - defaultStatsPort = 1936 - defaultHealthzPort = defaultStatsPort + // Default stats port. + defaultStatsPort = 1936 ) // NewCmdRouter implements the OpenShift CLI router command. @@ -436,21 +435,21 @@ func generateSecretsConfig(cfg *RouterConfig, namespace string, defaultCert []by return secrets, volumes, mounts, nil } -func generateProbeConfigForRouter(cfg *RouterConfig, ports []kapi.ContainerPort) *kapi.Probe { +func generateProbeConfigForRouter(path string, cfg *RouterConfig, ports []kapi.ContainerPort) *kapi.Probe { var probe *kapi.Probe if cfg.Type == "haproxy-router" { probe = &kapi.Probe{} - healthzPort := defaultHealthzPort + probePort := defaultStatsPort if cfg.StatsPort > 0 { - healthzPort = cfg.StatsPort + probePort = cfg.StatsPort } probe.Handler.HTTPGet = &kapi.HTTPGetAction{ - Path: "/healthz", + Path: path, Port: intstr.IntOrString{ Type: intstr.Int, - IntVal: int32(healthzPort), + IntVal: int32(probePort), }, } @@ -466,7 +465,7 @@ func generateProbeConfigForRouter(cfg *RouterConfig, ports []kapi.ContainerPort) } func generateLivenessProbeConfig(cfg *RouterConfig, ports []kapi.ContainerPort) *kapi.Probe { - probe := generateProbeConfigForRouter(cfg, ports) + probe := generateProbeConfigForRouter("/healthz", cfg, ports) if probe != nil { probe.InitialDelaySeconds = 10 } @@ -474,7 +473,7 @@ func generateLivenessProbeConfig(cfg *RouterConfig, ports []kapi.ContainerPort) } func generateReadinessProbeConfig(cfg *RouterConfig, ports []kapi.ContainerPort) *kapi.Probe { - probe := generateProbeConfigForRouter(cfg, ports) + probe := generateProbeConfigForRouter("healthz/ready", cfg, ports) if probe != nil { probe.InitialDelaySeconds = 10 } diff --git a/pkg/router/metrics/health.go b/pkg/router/metrics/health.go index dc99dbada8cd..3faebf1e0561 100644 --- a/pkg/router/metrics/health.go +++ b/pkg/router/metrics/health.go @@ -12,6 +12,7 @@ import ( "github.com/golang/glog" + templateplugin "github.com/openshift/origin/pkg/router/template" "k8s.io/apiserver/pkg/server/healthz" "k8s.io/kubernetes/pkg/probe" probehttp "k8s.io/kubernetes/pkg/probe/http" @@ -35,6 +36,28 @@ func HTTPBackendAvailable(u *url.URL) healthz.HealthzChecker { }) } +// HasSynced returns a healthz check that verifies the router has been synced at least +// once. +func HasSynced(router **templateplugin.TemplatePlugin) healthz.HealthzChecker { + return healthz.NamedCheck("has-synced", func(r *http.Request) error { + if router != nil { + if (*router).Router.SyncedAtLeastOnce() == true { + return nil + } else { + return fmt.Errorf("Router not synced") + } + } + return nil + }) +} + +func ControllerLive() healthz.HealthzChecker { + return healthz.NamedCheck("controller", func(r *http.Request) error { + return nil + }) + +} + // ProxyProtocolHTTPBackendAvailable returns a healthz check that verifies a backend supporting // the HAProxy PROXY protocol responds to a GET to the provided URL with 2xx or 3xx response. func ProxyProtocolHTTPBackendAvailable(u *url.URL) healthz.HealthzChecker { diff --git a/pkg/router/metrics/metrics.go b/pkg/router/metrics/metrics.go index 840c72bd2fe2..1f27afb30260 100644 --- a/pkg/router/metrics/metrics.go +++ b/pkg/router/metrics/metrics.go @@ -31,12 +31,14 @@ type Listener struct { Authorizer authorizer.Authorizer Record authorizer.AttributesRecord - Checks []healthz.HealthzChecker + LiveChecks []healthz.HealthzChecker + ReadyChecks []healthz.HealthzChecker } func (l Listener) handler() http.Handler { mux := http.NewServeMux() - healthz.InstallHandler(mux, l.Checks...) + healthz.InstallHandler(mux, l.LiveChecks...) + healthz.InstallPathHandler(mux, "/healthz/ready", l.ReadyChecks...) if l.Authenticator != nil { protected := http.NewServeMux()