Skip to content

Commit 4e56daf

Browse files
smarterclaytonsmarterclayton
committed
Introduce some default log verbosity control
Move a lot of common error logging into better buckets: glog.Errorf() - Always an error glog.Warningf() - Something unexpected, but probably not an error glog.V(0) - Generally useful for this to ALWAYS be visible to an operator * Programmer errors * Logging extra info about a panic * CLI argument handling glog.V(1) - A reasonable default log level if you don't want verbosity * Information about config (listening on X, watching Y) * Errors that repeat frequently that relate to conditions that can be corrected (pod detected as unhealthy) glog.V(2) - Useful steady state information about the service * Logging HTTP requests and their exit code * System state changing (killing pod) * Controller state change events (starting pods) * Scheduler log messages glog.V(3) - Extended information about changes * More info about system state changes glog.V(4) - Debug level verbosity (for now) * Logging in particularly thorny parts of code where you may want to come back later and check it
1 parent 74db9a1 commit 4e56daf

File tree

20 files changed

+97
-76
lines changed

20 files changed

+97
-76
lines changed

‎docs/logging.md

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
Logging Conventions
2+
===================
3+
4+
The following conventions for the glog levels to use. glog is globally prefered to "log" for better runtime control.
5+
6+
* glog.Errorf() - Always an error
7+
* glog.Warningf() - Something unexpected, but probably not an error
8+
* glog.Infof / glog.V(0) - Generally useful for this to ALWAYS be visible to an operator
9+
* Programmer errors
10+
* Logging extra info about a panic
11+
* CLI argument handling
12+
* glog.V(1) - A reasonable default log level if you don't want verbosity.
13+
* Information about config (listening on X, watching Y)
14+
* Errors that repeat frequently that relate to conditions that can be corrected (pod detected as unhealthy)
15+
* glog.V(2) - Useful steady state information about the service and important log messages that may correlate to significant changes in the system. This is the recommended default log level for most systems.
16+
* Logging HTTP requests and their exit code
17+
* System state changing (killing pod)
18+
* Controller state change events (starting pods)
19+
* Scheduler log messages
20+
* glog.V(3) - Extended information about changes
21+
* More info about system state changes
22+
* glog.V(4) - Debug level verbosity (for now)
23+
* Logging in particularly thorny parts of code where you may want to come back later and check it
24+
25+
As per the comments, the practical default level is V(2). Developers and QE environments may wish to run at V(3) or V(4).

‎pkg/client/cache/reflector.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,6 @@ func (r *Reflector) watchHandler(w watch.Interface, resourceVersion *uint64) err
166166
glog.Errorf("unexpected watch close - watch lasted less than a second and no items received")
167167
return errors.New("very short watch")
168168
}
169-
glog.Infof("watch close - %v total items received", eventCount)
169+
glog.V(4).Infof("watch close - %v total items received", eventCount)
170170
return nil
171171
}

‎pkg/cloudprovider/plugins.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ func RegisterCloudProvider(name string, cloud Factory) {
4242
if found {
4343
glog.Fatalf("Cloud provider %q was registered twice", name)
4444
}
45-
glog.Infof("Registered cloud provider %q", name)
45+
glog.V(1).Infof("Registered cloud provider %q", name)
4646
providers[name] = cloud
4747
}
4848

‎pkg/controller/replication_controller.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ func (rm *ReplicationManager) watchControllers(resourceVersion *uint64) {
115115
// that called us call us again.
116116
return
117117
}
118-
glog.Infof("Got watch: %#v", event)
118+
glog.V(4).Infof("Got watch: %#v", event)
119119
rc, ok := event.Object.(*api.ReplicationController)
120120
if !ok {
121121
glog.Errorf("unexpected object: %#v", event.Object)
@@ -125,7 +125,7 @@ func (rm *ReplicationManager) watchControllers(resourceVersion *uint64) {
125125
*resourceVersion = rc.ResourceVersion + 1
126126
// Sync even if this is a deletion event, to ensure that we leave
127127
// it in the desired state.
128-
glog.Infof("About to sync from watch: %v", rc.ID)
128+
glog.V(4).Infof("About to sync from watch: %v", rc.ID)
129129
rm.syncHandler(*rc)
130130
}
131131
}
@@ -153,7 +153,7 @@ func (rm *ReplicationManager) syncReplicationController(controllerSpec api.Repli
153153
diff *= -1
154154
wait := sync.WaitGroup{}
155155
wait.Add(diff)
156-
glog.Infof("Too few replicas, creating %d\n", diff)
156+
glog.V(2).Infof("Too few replicas, creating %d\n", diff)
157157
for i := 0; i < diff; i++ {
158158
go func() {
159159
defer wait.Done()
@@ -162,7 +162,7 @@ func (rm *ReplicationManager) syncReplicationController(controllerSpec api.Repli
162162
}
163163
wait.Wait()
164164
} else if diff > 0 {
165-
glog.Infof("Too many replicas, deleting %d\n", diff)
165+
glog.V(2).Infof("Too many replicas, deleting %d\n", diff)
166166
wait := sync.WaitGroup{}
167167
wait.Add(diff)
168168
for i := 0; i < diff; i++ {
@@ -191,7 +191,7 @@ func (rm *ReplicationManager) synchronize() {
191191
for ix := range controllerSpecs {
192192
go func(ix int) {
193193
defer wg.Done()
194-
glog.Infof("periodic sync of %v", controllerSpecs[ix].ID)
194+
glog.V(4).Infof("periodic sync of %v", controllerSpecs[ix].ID)
195195
err := rm.syncHandler(controllerSpecs[ix])
196196
if err != nil {
197197
glog.Errorf("Error synchronizing: %#v", err)

‎pkg/httplog/log.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ func (rl *respLogger) Addf(format string, data ...interface{}) {
148148
// Log is intended to be called once at the end of your request handler, via defer
149149
func (rl *respLogger) Log() {
150150
latency := time.Since(rl.startTime)
151-
glog.Infof("%s %s: (%v) %v%v%v", rl.req.Method, rl.req.RequestURI, latency, rl.status, rl.statusStack, rl.addedInfo)
151+
glog.V(2).Infof("%s %s: (%v) %v%v%v", rl.req.Method, rl.req.RequestURI, latency, rl.status, rl.statusStack, rl.addedInfo)
152152
}
153153

154154
// Header implements http.ResponseWriter.

‎pkg/kubelet/config/config.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -168,9 +168,9 @@ func (s *podStorage) merge(source string, change interface{}) (adds, updates, de
168168
switch update.Op {
169169
case kubelet.ADD, kubelet.UPDATE:
170170
if update.Op == kubelet.ADD {
171-
glog.Infof("Adding new pods from source %s : %v", source, update.Pods)
171+
glog.V(4).Infof("Adding new pods from source %s : %v", source, update.Pods)
172172
} else {
173-
glog.Infof("Updating pods from source %s : %v", source, update.Pods)
173+
glog.V(4).Infof("Updating pods from source %s : %v", source, update.Pods)
174174
}
175175

176176
filtered := filterInvalidPods(update.Pods, source)
@@ -193,7 +193,7 @@ func (s *podStorage) merge(source string, change interface{}) (adds, updates, de
193193
}
194194

195195
case kubelet.REMOVE:
196-
glog.Infof("Removing a pod %v", update)
196+
glog.V(4).Infof("Removing a pod %v", update)
197197
for _, value := range update.Pods {
198198
name := value.Name
199199
if existing, found := pods[name]; found {
@@ -206,7 +206,7 @@ func (s *podStorage) merge(source string, change interface{}) (adds, updates, de
206206
}
207207

208208
case kubelet.SET:
209-
glog.Infof("Setting pods for source %s : %v", source, update)
209+
glog.V(4).Infof("Setting pods for source %s : %v", source, update)
210210
// Clear the old map entries by just creating a new map
211211
oldPods := pods
212212
pods = make(map[string]*kubelet.Pod)
@@ -238,7 +238,7 @@ func (s *podStorage) merge(source string, change interface{}) (adds, updates, de
238238
}
239239

240240
default:
241-
glog.Infof("Received invalid update type: %v", update)
241+
glog.Warningf("Received invalid update type: %v", update)
242242

243243
}
244244

@@ -259,7 +259,7 @@ func filterInvalidPods(pods []kubelet.Pod, source string) (filtered []*kubelet.P
259259
errors = append(errors, errs...)
260260
}
261261
if len(errors) > 0 {
262-
glog.Warningf("Pod %d from %s failed validation, ignoring: %v", i+1, source, errors)
262+
glog.Warningf("Pod %d (%s) from %s failed validation, ignoring: %v", i+1, pods[i].Name, source, errors)
263263
continue
264264
}
265265
filtered = append(filtered, &pods[i])

‎pkg/kubelet/config/etcd.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ func NewSourceEtcd(key string, client tools.EtcdClient, updates chan<- interface
5454
helper: helper,
5555
updates: updates,
5656
}
57-
glog.Infof("Watching etcd for %s", key)
57+
glog.V(1).Infof("Watching etcd for %s", key)
5858
go util.Forever(source.run, time.Second)
5959
return source
6060
}
@@ -78,7 +78,7 @@ func (s *SourceEtcd) run() {
7878
continue
7979
}
8080

81-
glog.Infof("Received state from etcd watch: %+v", pods)
81+
glog.V(4).Infof("Received state from etcd watch: %+v", pods)
8282
s.updates <- kubelet.PodUpdate{pods, kubelet.SET}
8383
}
8484
}

‎pkg/kubelet/config/file.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ func NewSourceFile(path string, period time.Duration, updates chan<- interface{}
4545
path: path,
4646
updates: updates,
4747
}
48-
glog.Infof("Watching file %s", path)
48+
glog.V(1).Infof("Watching file %s", path)
4949
go util.Forever(config.run, period)
5050
return config
5151
}

‎pkg/kubelet/config/http.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ func NewSourceURL(url string, period time.Duration, updates chan<- interface{})
4444
updates: updates,
4545
data: nil,
4646
}
47-
glog.Infof("Watching URL %s", url)
47+
glog.V(1).Infof("Watching URL %s", url)
4848
go util.Forever(config.run, period)
4949
return config
5050
}

‎pkg/kubelet/dockertools/docker.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,11 +75,11 @@ func NewDockerPuller(client DockerInterface) DockerPuller {
7575
if err == nil {
7676
cfg.addToKeyring(dp.keyring)
7777
} else {
78-
glog.Errorf("Unable to parse docker config file: %v", err)
78+
glog.Errorf("Unable to parse Docker config file: %v", err)
7979
}
8080

8181
if dp.keyring.count() == 0 {
82-
glog.Infof("Continuing with empty docker keyring")
82+
glog.V(1).Infof("Continuing with empty Docker keyring")
8383
}
8484

8585
return dp
@@ -348,7 +348,7 @@ func ParseDockerName(name string) (podFullName, uuid, containerName string, hash
348348
var err error
349349
hash, err = strconv.ParseUint(pieces[1], 16, 32)
350350
if err != nil {
351-
glog.Infof("invalid container hash: %s", pieces[1])
351+
glog.Warningf("invalid container hash: %s", pieces[1])
352352
}
353353
}
354354
}

‎pkg/kubelet/kubelet.go

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,7 @@ func makePortsAndBindings(container *api.Container) (map[docker.Port]struct{}, m
220220
case "TCP":
221221
protocol = "/tcp"
222222
default:
223-
glog.Infof("Unknown protocol '%s': defaulting to TCP", port.Protocol)
223+
glog.Warningf("Unknown protocol '%s': defaulting to TCP", port.Protocol)
224224
protocol = "/tcp"
225225
}
226226
dockerPort := docker.Port(strconv.Itoa(interiorPort) + protocol)
@@ -345,7 +345,7 @@ func (kl *Kubelet) killContainer(dockerContainer *docker.APIContainers) error {
345345
}
346346

347347
func (kl *Kubelet) killContainerByID(ID, name string) error {
348-
glog.Infof("Killing: %s", ID)
348+
glog.V(2).Infof("Killing: %s", ID)
349349
err := kl.dockerClient.StopContainer(ID, 10)
350350
if len(name) == 0 {
351351
return err
@@ -425,7 +425,7 @@ func (kl *Kubelet) syncPod(pod *Pod, dockerContainers dockertools.DockerContaine
425425
if networkDockerContainer, found, _ := dockerContainers.FindPodContainer(podFullName, uuid, networkContainerName); found {
426426
netID = dockertools.DockerID(networkDockerContainer.ID)
427427
} else {
428-
glog.Infof("Network container doesn't exist, creating")
428+
glog.V(3).Infof("Network container doesn't exist, creating")
429429
count, err := kl.deleteAllContainers(pod, podFullName, dockerContainers)
430430
if err != nil {
431431
return err
@@ -468,7 +468,7 @@ func (kl *Kubelet) syncPod(pod *Pod, dockerContainers dockertools.DockerContaine
468468
expectedHash := dockertools.HashContainer(&container)
469469
if dockerContainer, found, hash := dockerContainers.FindPodContainer(podFullName, uuid, container.Name); found {
470470
containerID := dockertools.DockerID(dockerContainer.ID)
471-
glog.V(1).Infof("pod %s container %s exists as %v", podFullName, container.Name, containerID)
471+
glog.V(3).Infof("pod %s container %s exists as %v", podFullName, container.Name, containerID)
472472

473473
// look for changes in the container.
474474
if hash == 0 || hash == expectedHash {
@@ -485,7 +485,7 @@ func (kl *Kubelet) syncPod(pod *Pod, dockerContainers dockertools.DockerContaine
485485
}
486486
glog.V(1).Infof("pod %s container %s is unhealthy.", podFullName, container.Name, healthy)
487487
} else {
488-
glog.V(1).Infof("container hash changed %d vs %d.", hash, expectedHash)
488+
glog.V(3).Infof("container hash changed %d vs %d.", hash, expectedHash)
489489
}
490490
if err := kl.killContainer(dockerContainer); err != nil {
491491
glog.V(1).Infof("Failed to kill container %s: %v", dockerContainer.ID, err)
@@ -503,21 +503,21 @@ func (kl *Kubelet) syncPod(pod *Pod, dockerContainers dockertools.DockerContaine
503503

504504
if len(recentContainers) > 0 && pod.Manifest.RestartPolicy.Always == nil {
505505
if pod.Manifest.RestartPolicy.Never != nil {
506-
glog.Infof("Already ran container with name %s--%s--%s, do nothing",
506+
glog.V(3).Infof("Already ran container with name %s--%s--%s, do nothing",
507507
podFullName, uuid, container.Name)
508508
continue
509509
}
510510
if pod.Manifest.RestartPolicy.OnFailure != nil {
511511
// Check the exit code of last run
512512
if recentContainers[0].State.ExitCode == 0 {
513-
glog.Infof("Already successfully ran container with name %s--%s--%s, do nothing",
513+
glog.V(3).Infof("Already successfully ran container with name %s--%s--%s, do nothing",
514514
podFullName, uuid, container.Name)
515515
continue
516516
}
517517
}
518518
}
519519

520-
glog.Infof("Container with name %s--%s--%s doesn't exist, creating %#v", podFullName, uuid, container.Name, container)
520+
glog.V(3).Infof("Container with name %s--%s--%s doesn't exist, creating %#v", podFullName, uuid, container.Name, container)
521521
if err := kl.dockerPuller.Pull(container.Image); err != nil {
522522
glog.Errorf("Failed to pull image %s: %v skipping pod %s container %s.", container.Image, err, podFullName, container.Name)
523523
continue
@@ -579,11 +579,11 @@ func (kl *Kubelet) reconcileVolumes(pods []Pod) error {
579579
if _, ok := desiredVolumes[name]; !ok {
580580
//TODO (jonesdl) We should somehow differentiate between volumes that are supposed
581581
//to be deleted and volumes that are leftover after a crash.
582-
glog.Infof("Orphaned volume %s found, tearing down volume", name)
582+
glog.Warningf("Orphaned volume %s found, tearing down volume", name)
583583
//TODO (jonesdl) This should not block other kubelet synchronization procedures
584584
err := vol.TearDown()
585585
if err != nil {
586-
glog.Infof("Could not tear down volume %s (%s)", name, err)
586+
glog.Errorf("Could not tear down volume %s (%s)", name, err)
587587
}
588588
}
589589
}
@@ -592,7 +592,7 @@ func (kl *Kubelet) reconcileVolumes(pods []Pod) error {
592592

593593
// SyncPods synchronizes the configured list of pods (desired state) with the host current state.
594594
func (kl *Kubelet) SyncPods(pods []Pod) error {
595-
glog.Infof("Desired [%s]: %+v", kl.hostname, pods)
595+
glog.V(4).Infof("Desired [%s]: %+v", kl.hostname, pods)
596596
var err error
597597
desiredContainers := make(map[podContainer]empty)
598598

@@ -675,13 +675,13 @@ func (kl *Kubelet) syncLoop(updates <-chan PodUpdate, handler SyncHandler) {
675675
case u := <-updates:
676676
switch u.Op {
677677
case SET:
678-
glog.Infof("Containers changed [%s]", kl.hostname)
678+
glog.V(3).Infof("Containers changed [%s]", kl.hostname)
679679
pods = u.Pods
680680
pods = filterHostPortConflicts(pods)
681681

682682
case UPDATE:
683683
//TODO: implement updates of containers
684-
glog.Infof("Containers updated, not implemented [%s]", kl.hostname)
684+
glog.Warningf("Containers updated, not implemented [%s]", kl.hostname)
685685
continue
686686

687687
default:

‎pkg/kubelet/server.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ type Server struct {
4848

4949
// ListenAndServeKubeletServer initializes a server to respond to HTTP network requests on the Kubelet.
5050
func ListenAndServeKubeletServer(host HostInterface, updates chan<- interface{}, address string, port uint) {
51-
glog.Infof("Starting to listen on %s:%d", address, port)
51+
glog.V(1).Infof("Starting to listen on %s:%d", address, port)
5252
handler := NewServer(host, updates)
5353
s := &http.Server{
5454
Addr: net.JoinHostPort(address, strconv.FormatUint(uint64(port), 10)),

‎pkg/proxy/config/api.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ func handleServicesWatch(resourceVersion *uint64, ch <-chan watch.Event, updates
101101
select {
102102
case event, ok := <-ch:
103103
if !ok {
104-
glog.V(2).Infof("WatchServices channel closed")
104+
glog.V(4).Infof("WatchServices channel closed")
105105
return
106106
}
107107

@@ -150,7 +150,7 @@ func handleEndpointsWatch(resourceVersion *uint64, ch <-chan watch.Event, update
150150
select {
151151
case event, ok := <-ch:
152152
if !ok {
153-
glog.V(2).Infof("WatchEndpoints channel closed")
153+
glog.V(4).Infof("WatchEndpoints channel closed")
154154
return
155155
}
156156

‎pkg/proxy/config/config.go

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -125,24 +125,24 @@ func (s *endpointsStore) Merge(source string, change interface{}) error {
125125
update := change.(EndpointsUpdate)
126126
switch update.Op {
127127
case ADD:
128-
glog.Infof("Adding new endpoint from source %s : %v", source, update.Endpoints)
128+
glog.V(4).Infof("Adding new endpoint from source %s : %v", source, update.Endpoints)
129129
for _, value := range update.Endpoints {
130130
endpoints[value.ID] = value
131131
}
132132
case REMOVE:
133-
glog.Infof("Removing an endpoint %v", update)
133+
glog.V(4).Infof("Removing an endpoint %v", update)
134134
for _, value := range update.Endpoints {
135135
delete(endpoints, value.ID)
136136
}
137137
case SET:
138-
glog.Infof("Setting endpoints %v", update)
138+
glog.V(4).Infof("Setting endpoints %v", update)
139139
// Clear the old map entries by just creating a new map
140140
endpoints = make(map[string]api.Endpoints)
141141
for _, value := range update.Endpoints {
142142
endpoints[value.ID] = value
143143
}
144144
default:
145-
glog.Infof("Received invalid update type: %v", update)
145+
glog.V(4).Infof("Received invalid update type: %v", update)
146146
}
147147
s.endpoints[source] = endpoints
148148
s.endpointLock.Unlock()
@@ -220,24 +220,24 @@ func (s *serviceStore) Merge(source string, change interface{}) error {
220220
update := change.(ServiceUpdate)
221221
switch update.Op {
222222
case ADD:
223-
glog.Infof("Adding new service from source %s : %v", source, update.Services)
223+
glog.V(4).Infof("Adding new service from source %s : %v", source, update.Services)
224224
for _, value := range update.Services {
225225
services[value.ID] = value
226226
}
227227
case REMOVE:
228-
glog.Infof("Removing a service %v", update)
228+
glog.V(4).Infof("Removing a service %v", update)
229229
for _, value := range update.Services {
230230
delete(services, value.ID)
231231
}
232232
case SET:
233-
glog.Infof("Setting services %v", update)
233+
glog.V(4).Infof("Setting services %v", update)
234234
// Clear the old map entries by just creating a new map
235235
services = make(map[string]api.Service)
236236
for _, value := range update.Services {
237237
services[value.ID] = value
238238
}
239239
default:
240-
glog.Infof("Received invalid update type: %v", update)
240+
glog.V(4).Infof("Received invalid update type: %v", update)
241241
}
242242
s.services[source] = services
243243
s.serviceLock.Unlock()

0 commit comments

Comments
 (0)