Skip to content

Commit ca34c4f

Browse files
committed
fix leader election bug
1 parent e28a842 commit ca34c4f

File tree

1 file changed

+44
-51
lines changed

1 file changed

+44
-51
lines changed

pkg/manager/internal.go

Lines changed: 44 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -351,6 +351,39 @@ func (cm *controllerManager) Start(ctx context.Context) (err error) {
351351
// Initialize the internal context.
352352
cm.internalCtx, cm.internalCancel = context.WithCancel(ctx)
353353

354+
leaderElector, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{
355+
Lock: cm.resourceLock,
356+
LeaseDuration: cm.leaseDuration,
357+
RenewDeadline: cm.renewDeadline,
358+
RetryPeriod: cm.retryPeriod,
359+
Callbacks: leaderelection.LeaderCallbacks{
360+
OnStartedLeading: func(_ context.Context) {
361+
if err := cm.startLeaderElectionRunnables(); err != nil {
362+
cm.errChan <- err
363+
return
364+
}
365+
close(cm.elected)
366+
},
367+
OnStoppedLeading: func() {
368+
if cm.onStoppedLeading != nil {
369+
cm.onStoppedLeading()
370+
}
371+
// Make sure graceful shutdown is skipped if we lost the leader lock without
372+
// intending to.
373+
cm.gracefulShutdownTimeout = time.Duration(0)
374+
// Most implementations of leader election log.Fatal() here.
375+
// Since Start is wrapped in log.Fatal when called, we can just return
376+
// an error here which will cause the program to exit.
377+
cm.errChan <- errors.New("leader election lost")
378+
},
379+
},
380+
ReleaseOnCancel: cm.leaderElectionReleaseOnCancel,
381+
Name: cm.leaderElectionID,
382+
})
383+
if err != nil {
384+
return err
385+
}
386+
354387
// This chan indicates that stop is complete, in other words all runnables have returned or timeout on stop request
355388
stopComplete := make(chan struct{})
356389
defer close(stopComplete)
@@ -433,19 +466,22 @@ func (cm *controllerManager) Start(ctx context.Context) (err error) {
433466
{
434467
ctx, cancel := context.WithCancel(context.Background())
435468
cm.leaderElectionCancel = cancel
436-
go func() {
437-
if cm.resourceLock != nil {
438-
if err := cm.startLeaderElection(ctx); err != nil {
439-
cm.errChan <- err
440-
}
441-
} else {
469+
if cm.resourceLock != nil {
470+
// Start the leader elector process
471+
go func() {
472+
leaderElector.Run(ctx)
473+
<-ctx.Done()
474+
close(cm.leaderElectionStopped)
475+
}()
476+
} else {
477+
go func() {
442478
// Treat not having leader election enabled the same as being elected.
443479
if err := cm.startLeaderElectionRunnables(); err != nil {
444480
cm.errChan <- err
445481
}
446482
close(cm.elected)
447-
}
448-
}()
483+
}()
484+
}
449485
}
450486

451487
ready = true
@@ -568,49 +604,6 @@ func (cm *controllerManager) startLeaderElectionRunnables() error {
568604
return cm.runnables.LeaderElection.Start(cm.internalCtx)
569605
}
570606

571-
func (cm *controllerManager) startLeaderElection(ctx context.Context) (err error) {
572-
l, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{
573-
Lock: cm.resourceLock,
574-
LeaseDuration: cm.leaseDuration,
575-
RenewDeadline: cm.renewDeadline,
576-
RetryPeriod: cm.retryPeriod,
577-
Callbacks: leaderelection.LeaderCallbacks{
578-
OnStartedLeading: func(_ context.Context) {
579-
if err := cm.startLeaderElectionRunnables(); err != nil {
580-
cm.errChan <- err
581-
return
582-
}
583-
close(cm.elected)
584-
},
585-
OnStoppedLeading: func() {
586-
if cm.onStoppedLeading != nil {
587-
cm.onStoppedLeading()
588-
}
589-
// Make sure graceful shutdown is skipped if we lost the leader lock without
590-
// intending to.
591-
cm.gracefulShutdownTimeout = time.Duration(0)
592-
// Most implementations of leader election log.Fatal() here.
593-
// Since Start is wrapped in log.Fatal when called, we can just return
594-
// an error here which will cause the program to exit.
595-
cm.errChan <- errors.New("leader election lost")
596-
},
597-
},
598-
ReleaseOnCancel: cm.leaderElectionReleaseOnCancel,
599-
Name: cm.leaderElectionID,
600-
})
601-
if err != nil {
602-
return err
603-
}
604-
605-
// Start the leader elector process
606-
go func() {
607-
l.Run(ctx)
608-
<-ctx.Done()
609-
close(cm.leaderElectionStopped)
610-
}()
611-
return nil
612-
}
613-
614607
func (cm *controllerManager) Elected() <-chan struct{} {
615608
return cm.elected
616609
}

0 commit comments

Comments
 (0)