External Admission Webhooks

作用及使用场景

当需要对某些api请求或者所有请求进行校验或者修改object的时候，可以考虑使用ValidatingAdmissionWebhook或者MutatingAdmissionWebhook，两者的区别：

  • ValidatingAdmissionWebhook不允许在webhook中对Object进行修改，只是返回结果是true或false
  • MutatingAdmissionWebhook运行在webhook中对Object进行修改

启用

在1.10之前的版本，需要使用 --admission-control 启用相关配置，并且是按配置的顺序来决定运行顺序，因此需要用户对于Admission Controllers 完全了解。

在1.10之后的版本，上述配置已经废弃，建议使用--enable-admission-plugins=MutatingAdmissionWebhook,ValidatingAdmissionWebhook，并且用户指定的顺序并不会影响实际运行顺序，更加友好。

官方的Using Admission Controllers - Kubernetes有关于这方面的详细说明。

流程

kube-apiserver –> 认证鉴权 –> Admission Controller –> webhook

使用

创建admission服务，以供kube-apiserver调用

package main

import (
	"crypto/tls"
	"encoding/json"
	"flag"
	"io/ioutil"
	"net/http"

	admissionv1beta1 "k8s.io/api/admission/v1beta1"
	admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/klog"
)

type patchOperation struct {
	Op    string      `json:"op"`
	Path  string      `json:"path"`
	Value interface{} `json:"value,omitempty"`
}

var scheme = runtime.NewScheme()
var codecs = serializer.NewCodecFactory(scheme)

func init() {
	addToScheme(scheme)
}

func addToScheme(scheme *runtime.Scheme) {
	utilruntime.Must(corev1.AddToScheme(scheme))
	utilruntime.Must(admissionv1beta1.AddToScheme(scheme))
	utilruntime.Must(admissionregistrationv1beta1.AddToScheme(scheme))
}

func toAdmissionResponse(err error) *admissionv1beta1.AdmissionResponse {
	return &admissionv1beta1.AdmissionResponse{
		Result: &metav1.Status{
			Message: err.Error(),
		},
	}
}

func updateAnnotation(target map[string]string, added map[string]string) (patch []patchOperation) {
	if target == nil {
		klog.Info(target)
		target = map[string]string{}
	}
	for key, value := range added {
		if target[key] == "" {
			patch = append(patch, patchOperation{
				Op:    "add",
				Path:  "/metadata/annotations/" + key,
				Value: value,
			})
		} else {
			patch = append(patch, patchOperation{
				Op:    "replace",
				Path:  "/metadata/annotations/" + key,
				Value: value,
			})
		}
	}
	return patch
}

func mutatePods(ar admissionv1beta1.AdmissionReview) *admissionv1beta1.AdmissionResponse {
	klog.V(2).Info("Inject pods.")

	podResource := metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"}
	if ar.Request.Resource != podResource {
		klog.Error("expect resource to be %s", podResource)
		return nil
	}

	raw := ar.Request.Object.Raw
	pod := corev1.Pod{}
	if _, _, err := codecs.UniversalDeserializer().Decode(raw, nil, &pod); err != nil {
		klog.Error(err)
		return toAdmissionResponse(err)
	}

	reviewResponse := admissionv1beta1.AdmissionResponse{}
	reviewResponse.Allowed = true

	klog.V(3).Infof("Pod %s annotations: %v", pod.Name, pod.Annotations)

	if pod.Annotations["injector.test.io/inject"] != "true" {
		return &reviewResponse
	}

	var patch []patchOperation

	// patch = append(patch, patchOperation{Op: "add", Path: "/metadata/annotations", Value: map[string]string{"injector.test.io/status": "success"}})
	added := map[string]string{"injector.test.io.status": "success"}
	patch = append(patch, updateAnnotation(pod.Annotations, added)...)
	if b, err := json.Marshal(patch); err == nil {
		pt := admissionv1beta1.PatchTypeJSONPatch
		reviewResponse.PatchType = &pt
		reviewResponse.Patch = b
	}

	return &reviewResponse
}

func injectHandler(w http.ResponseWriter, r *http.Request) {
	var body []byte
	if r.Body != nil {
		if data, err := ioutil.ReadAll(r.Body); err == nil {
			body = data
		}
	}

	// verify the content type is accurate
	contentType := r.Header.Get("Content-Type")
	if contentType != "application/json" {
		klog.Errorf("contentType=%s, expect application/json", contentType)
		return
	}

	requestedAdmissionReview := admissionv1beta1.AdmissionReview{}
	responseAdmissionReview := admissionv1beta1.AdmissionReview{}

	deserializer := codecs.UniversalDeserializer()
	if _, _, err := deserializer.Decode(body, nil, &requestedAdmissionReview); err != nil {
		klog.Error(err)
		responseAdmissionReview.Response = toAdmissionResponse(err)
	} else {
		responseAdmissionReview.Response = mutatePods(requestedAdmissionReview)
	}

	responseAdmissionReview.Response.UID = requestedAdmissionReview.Request.UID
	klog.V(1).Infof("sending response: %v", responseAdmissionReview.Response)

	respBytes, err := json.Marshal(responseAdmissionReview)
	if err != nil {
		klog.Error(err)
	}
	if _, err := w.Write(respBytes); err != nil {
		klog.Error(err)
	}
}

func configTLS(certFile, keyFile string) (*tls.Config, error) {
	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
	if err != nil {
		return nil, err
	}

	tlsConfig := &tls.Config{
		Certificates:       []tls.Certificate{cert},
		InsecureSkipVerify: true,
	}
	return tlsConfig, nil
}

var (
	certFile string
	keyFile  string
)

func main() {
	klog.InitFlags(nil)
	flag.Set("logtostderr", "true")

	flag.StringVar(&certFile, "tls-cert-file", "", "File containing the default x509 Certificate for HTTPS.")
	flag.StringVar(&keyFile, "tls-key-file", "", "File containing the default x509 private key matching --tls-cert-file.")

	flag.Parse()
	defer klog.Flush()
	// cfg, err := configTLS(certFile, keyFile)
	// if err != nil {
	// 	klog.Fatalf("failed to tls config with %v", err)
	// }

	http.HandleFunc("/pods/inject", injectHandler)

	srv := &http.Server{
		Addr: ":9443",
		TLSConfig: &tls.Config{
			InsecureSkipVerify: true,
		},
	}
	klog.Info("Admission server starting...")
	srv.ListenAndServeTLS(certFile, keyFile)

}

创建上述服务所使用的证书

  • CA
[root@k8s-1 ~]$ cat ca-config.json
{
    "signing": {
        "default": {
            "expiry": "43800h"
        },
        "profiles": {
            "server": {
                "expiry": "43800h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth"
                ]
            },
            "client": {
                "expiry": "43800h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "client auth"
                ]
            }
        }
    }
}

[root@k8s-1 ~]$ cat ca-csr.json
{
  "CN": "cloud.kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "ZheJiang",
      "L": "HangZhou",
      "O": "k8s",
      "OU": "cloud"
    }
  ]
}

cfssl gencert -initca ca-config.json | cfssljson -bare ca

  • Server
[root@k8s-1 ~]$ cat server.json
{
    "CN": "cloud.server",
    "hosts": [
        "127.0.0.1",
        "${IP}",
        "*.cloud"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "China",
            "L": "HangZhou",
            "ST": "ZheJiang"
        }
    ]
}

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server server.json | cfssljson -bare server

  • Client
[root@k8s-1 ~]$ cat client.json
{
    "CN": "cloud.client",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "China",
            "L": "HangZhou",
            "ST": "ZheJiang"
        }
    ]
}

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client cient.json | cfssljson -bare client

创建admission文件

apiVersion: admissionregistration.k8s.io/v1beta1
kind: MutatingWebhookConfiguration
metadata:
  name: test-webhook
  namespace: kube-system
  labels:
    app: admission-webhook-injector
webhooks:
- name: sidecar-injector.test.io
  rules:
  - apiGroups: [""]
    apiVersions: ["v1"]
    operations: ["CREATE"]
    resources: ["pods"]
  clientConfig:
    url: "https://${IP}:9443/pods/inject"
    caBundle: <pem encoded ca cert that signs the server cert used by the webhook>

命令： cat ca.pem | base64 | tr -d '\n'

Testing

启动admission服务

./admission -tls-cert-file certs/server.pem -tls-key-file certs/server-key.pem -v 3

创建测试pod的yaml文件

apiVersion: v1
kind: Pod
metadata:
  annotations:
    injector.test.io/inject: "true"
  labels:
    app: testpod
  name: testpod
spec:
  containers:
  - image: k8s.gcr.io/etcd
    imagePullPolicy: IfNotPresent
    name: etcd

kubectl apply -f pod.yaml

查看新生成的pod，发现annotations有"injector.test.io.status": "success"，说明admission生效

Kubernetes 准入控制 Admission Controller 介绍 | Coder·码农网

深度剖析Kubernetes动态准入控制之Admission Webhooks - WaltonWang’s Blog - 开源中国

Dynamic Admission Control - Kubernetes