Use the leader election mechanism in kubernetes to complete your own ha application

Backgroud

The previous chapter in , Yes kubernetes In-depth analysis of the election principle , Here is a example To achieve a , utilize kubernetes The high availability application completed by the provided election mechanism .

For this chapter, you need to understand some concepts in advance before you can continue to read

• leader election mechanism
• RBCA
• Pod runtime mechanism

Implementation

Code implementation

If you just use Kubernetes In the lock , The implementation code is only a few lines .

package main

import (
	"context"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"syscall"
	"time"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/tools/leaderelection"
	"k8s.io/client-go/tools/leaderelection/resourcelock"
	"k8s.io/klog/v2"
)

func buildConfig(kubeconfig string) (*rest.Config, error) {
    
	if kubeconfig != "" {
    
		cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
		if err != nil {
    
			return nil, err
		}
		return cfg, nil
	}

	cfg, err := rest.InClusterConfig()
	if err != nil {
    
		return nil, err
	}
	return cfg, nil
}

func main() {
    
	klog.InitFlags(nil)

	var kubeconfig string
	var leaseLockName string
	var leaseLockNamespace string
	var id string
	//  Initialize the client part 
	flag.StringVar(&kubeconfig, "kubeconfig", "", "absolute path to the kubeconfig file")
	flag.StringVar(&id, "id", "", "the holder identity name")
	flag.StringVar(&leaseLockName, "lease-lock-name", "", "the lease lock resource name")
	flag.StringVar(&leaseLockNamespace, "lease-lock-namespace", "", "the lease lock resource namespace")
	flag.Parse()

	if leaseLockName == "" {
    
		klog.Fatal("unable to get lease lock resource name (missing lease-lock-name flag).")
	}
	if leaseLockNamespace == "" {
    
		klog.Fatal("unable to get lease lock resource namespace (missing lease-lock-namespace flag).")
	}
	config, err := buildConfig(kubeconfig)
	if err != nil {
    
		klog.Fatal(err)
	}
	client := clientset.NewForConfigOrDie(config)

	run := func(ctx context.Context) {
    
		//  Implemented business logic , This is just an experiment , Just print it directly 
		klog.Info("Controller loop...")

		for {
    
			fmt.Println("I am leader, I was working.")
			time.Sleep(time.Second * 5)
		}
	}

	// use a Go context so we can tell the leaderelection code when we
	// want to step down
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	//  Monitoring system interrupt 
	ch := make(chan os.Signal, 1)
	signal.Notify(ch, os.Interrupt, syscall.SIGTERM)
	go func() {
    
		<-ch
		klog.Info("Received termination, signaling shutdown")
		cancel()
	}()

	//  Create a resource lock 
	lock := &resourcelock.LeaseLock{
    
		LeaseMeta: metav1.ObjectMeta{
    
			Name:      leaseLockName,
			Namespace: leaseLockNamespace,
		},
		Client: client.CoordinationV1(),
		LockConfig: resourcelock.ResourceLockConfig{
    
			Identity: id,
		},
	}

	//  Start an election cycle 
	leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
    
		Lock:            lock,
		ReleaseOnCancel: true,
		LeaseDuration:   60 * time.Second,
		RenewDeadline:   15 * time.Second,
		RetryPeriod:     5 * time.Second,
		Callbacks: leaderelection.LeaderCallbacks{
    
			OnStartedLeading: func(ctx context.Context) {
    
				//  When the election is leader The business logic that runs after 
				run(ctx)
			},
			OnStoppedLeading: func() {
    
				// we can do cleanup here
				klog.Infof("leader lost: %s", id)
				os.Exit(0)
			},
			OnNewLeader: func(identity string) {
     //  The act of applying for an election 
				if identity == id {
    
					return
				}
				klog.Infof("new leader elected: %s", identity)
			},
		},
	})
}

notes ： such lease The lock can only be in-cluster Run in mode , If you need a program like binary deployment , You can choose endpoint Type of resource lock .

Create a mirror image

The image has been created and uploaded to dockerhub（cylonchau/leaderelection:v0.0.2） Yes , If you just learn how it works , Ignore this step

FROM golang:alpine AS builder
MAINTAINER cylon
WORKDIR /election
COPY . /election
ENV GOPROXY https://goproxy.cn,direct
RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o elector main.go

FROM alpine AS runner
WORKDIR /go/elector
COPY --from=builder /election/elector .
VOLUME ["/election"]
ENTRYPOINT ["./elector"]

Prepare resource list

By default ,Kubernetes Running pod In the request Kubernetes Resources in the cluster , The default account does not have permission , The default service account does not have permission to access coordination API, So we need to create another serviceaccount And set it accordingly Corresponding RBAC Permission binding ; Configure this in the listing sa, All the pod You will have the authority to coordinate locks

apiVersion: v1
kind: ServiceAccount
metadata:
  name: sa-leaderelection
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: leaderelection
rules:
  - apiGroups:
      - coordination.k8s.io
    resources:
      - leases
    verbs:
      - '*'
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: leaderelection
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: leaderelection
subjects:
  - kind: ServiceAccount
    name: sa-leaderelection
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: leaderelection
  name: leaderelection
  namespace: default
spec:
  replicas: 3
  selector:
    matchLabels:
      app: leaderelection
  template:
    metadata:
      labels:
        app: leaderelection
    spec:
      containers:
        - image: cylonchau/leaderelection:v0.0.2
          imagePullPolicy: IfNotPresent
          command: ["./elector"]
          args:
          - "-id=$(POD_NAME)"
          - "-lease-lock-name=test"
          - "-lease-lock-namespace=default"
          env:
          - name: POD_NAME
            valueFrom:
              fieldRef:
                apiVersion: v1
                fieldPath: metadata.name
          name: elector
      serviceAccountName: sa-leaderelection

Running in cluster

After executing the list , When pod After starting , You can see that a lease

$ kubectl get lease
NAME   HOLDER                            AGE
test   leaderelection-5644c5f84f-frs5n   1s


$ kubectl describe lease
Name:         test
Namespace:    default
Labels:       <none>
Annotations:  <none>
API Version:  coordination.k8s.io/v1
Kind:         Lease
Metadata:
  Creation Timestamp:  2022-06-28T16:39:45Z
  Managed Fields:
    API Version:  coordination.k8s.io/v1
    Fields Type:  FieldsV1
    fieldsV1:
      f:spec:
        f:acquireTime:
        f:holderIdentity:
        f:leaseDurationSeconds:
        f:leaseTransitions:
        f:renewTime:
    Manager:         elector
    Operation:       Update
    Time:            2022-06-28T16:39:45Z
  Resource Version:  131693
  Self Link:         /apis/coordination.k8s.io/v1/namespaces/default/leases/test
  UID:               bef2b164-a117-44bd-bad3-3e651c94c97b
Spec:
  Acquire Time:            2022-06-28T16:39:45.931873Z
  Holder Identity:         leaderelection-5644c5f84f-frs5n
  Lease Duration Seconds:  60
  Lease Transitions:       0
  Renew Time:              2022-06-28T16:39:55.963537Z
Events:                    <none>

View the corresponding through the information of its holder pod（ Because in the program holder Identity The settings are pod The name of ）, It's actually working pod.

As described in the example above , It's using Kubernetes The cluster completes leader The plan for the election , Although this is not the perfect solution , But this is a simple way , Because it can be used without deploying more things on the cluster or doing a lot of code work Kubernetes Cluster to complete a highly available HA application .