Golang项目引入leaderelection实现leader选举

在k8s本身的组件中，kube-scheduler和kube-manager-controller两个组件是有leader选举的，这个选举机制是k8s对于这两个组件的高可用保障。即正常情况下kube-scheduler或kube-manager-controller组件的多个副本只有一个是处于业务逻辑运行状态，其它副本则不断的尝试去获取锁，去竞争leader，直到自己成为leader。如果正在运行的leader因某种原因导致当前进程退出，或者锁丢失，则由其它副本去竞争新的leader，获取leader继而执行业务逻辑。

不光是k8s本身组件用到了这个选举策略，我们自己定义的服务同样可以用这个算法去实现选主。在k8s client包中就提供了接口供用户使用。代码路径在client-go/tools/leaderelection下。

一、代码应用举例

1、引入leader选举代码demo

代码主要涉及到两个部分，一个是leader election的配置创建，一个leader election的启动。

其中leader election的配置创建主要有两个部分逻辑：

• resourcelock创建：resourclock的类型有leaselock、configmaplock和endpointlock三类。
• LeaderElectionConfig创建：主要是配置election所需要的信息，例如上面创建的lock、每个任期的时长（LeaseDuration）等。

leader election的启动主要也有两个部分逻辑：

• 设置leaderElection callbacks：

  • OnStartedLeading：在成功获取leader身份后调用的方法。
  • OnStoppedLeading: 当收到ctx.Done()，或者不再是leader身份时被调用的方法。
  • OnNewLeader：当产生新的leader后执行的方法。

• 通过leaderelection.RunOrDie开启leader选举。

package main

import (
	"context"
	"fmt"
	"math/rand"
	"strconv"
	"time"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	clientset "k8s.io/client-go/kubernetes"
	clientRest "k8s.io/client-go/rest"
	"k8s.io/client-go/tools/leaderelection"
	"k8s.io/client-go/tools/leaderelection/resourcelock"
)

var (
	// 租约（在k8s中称之为租约：lease）的名称和命名空间
	leaseLockName      = "demo-lock"
	leaseLockNamespace = "default"
)

func main() {
   
   
	fmt.Println("start leader election.....")

	//如果开启leader选举，只有抢到锁的服务才能启动任务
	run := func(context context.Context) {
   
   
		//比如这里启动一个定时任务
		fmt.Println("start cron task")
	}

	// 初始化k8s client
	cfg, err := clientRest.InClusterConfig()
	if err != nil {
   
   
		fmt.Printf("init cliconfig failed: %v", err)
		panic(err)
	}
	client := clientset.NewForConfigOrDie(cfg)

	ctxt, cancel := context.WithCancel(context.Background())
	defer cancel()
    
    // 用于接收进程停止信号
    ch := make(chan os.Signal, 1)
	signal.Notify(ch, os.Interrupt, syscall.SIGTERM)
	go func() {
   
   
		<-ch
		klog.Info("Received termination, signaling shutdown")
		cancel()
	}()

	// 生成随机id，作为租约持有者（即leader）的唯一标识
	identityId := strconv.Itoa(rand.Int())

	// 指定锁的资源对象，这里使用了Lease资源，还支持configmap，endpoint，或者multilock(即多种配合使用)
	//endpointsResourceLock  = "endpoints"
	//configMapsResourceLock = "configmaps"
	//LeasesResourceLock     = "leases"
	lock := &resourcelock.LeaseLock{
   
   
		LeaseMeta: metav1.ObjectMeta{
   
   
			Name:      leaseLockName,
			Namespace: leaseLockNamespace,
		},
		Client: client.CoordinationV1(),
		LockConfig: resourcelock.ResourceLockConfig{
   
   
			Identity: identityId,
		},
	}

	// leader选举核心代码
	leaderelection.RunOrDie(ctxt, leaderelection.LeaderElectionConfig{
   
   
		// Lock is the resource that will be used for locking
		Lock: lock,
		// LeaseDuration is the duration that non-leader candidates will wait to force acquire leadership.
		LeaseDuration: 75 * time.Second,  //租约时间
		// RenewDeadline is the duration that the acting master will retry refreshing leadership before giving up.
		RenewDeadline: 60 * time.Second,  //更新租约时间
		// RetryPeriod is the duration the LeaderElector clients should wait between tries of actions.
		RetryPeriod: 5 * time.Second,     //非leader节点重试时间
		// Callbacks are callbacks that are triggered during certain lifecycle events of the LeaderElector
		Callbacks: leaderelection.LeaderCallbacks{
   
   
			// OnStartedLeading is called when a LeaderElector client starts leading
			OnStartedLeading: run,               //变为leader执行的业务代码
			// OnStoppedLeading is called when a LeaderElector client stops leading
			OnStoppedLeading: func() {
   
              //退出leader执行退出
				cancel()
				fmt.Println("leader election lost")
			},
			// OnNewLeader is called when the client observes a leader that is
			// not the previously observed leader. This includes the first observed
			// leader when the client starts.
			OnNewLeader: func(identity string) {
   
     //当产生新的leader后执行的方法
				if identity == identityId {
   
   
					fmt.Printf("service self：%s is alreay hold the lock", identity)
					return
				}
				fmt.Printf("new leader elected: %s", identity)
			},
		},
	})
}

2、创建lease锁资源

创建对应的lease锁资源，用于上述demo服务竞争lease锁资源。

apiVersion: coordination.k8s.io/v1
kind: Lease
metadata:
  name: demo-lock
  namespace: default
spec:
    leaseDurationSeconds: 75

3、创建ClusterRole资源

创建ClusterRole资源，定义上述demo服务的角色，它有哪些权限。

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: demo-cluster-role
rules:
- apiGroups:
  - coordination.k8s.io
  resources:
  - leases
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - patch
  - delete

4、创建ClusterRoleBinding资源

给demo服务绑定对应的ClusterRole资源。

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: demo-cluster-role-binding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: demo-cluster-role
subjects:
- kind: ServiceAccount
  ## 服务名称demo
  name: demo
  namespace: default

5、验证

可以尝试开启多个demo服务，只有一个demo服务会竞争获取到lease锁资源，成为leader。

//日志输出：
start leader election.....
start cron task
new leader elected: 5206980002934563431
...

• 查看lease资源
  可以看到这个lease租约资源已经被id=5206980002934563431的demo服务竞争获取。

$ kubectl get lease
NAME           HOLDER                AGE
demo-lock   5206980002934563431