k8s资源限制--gatekeeper初探

OPA是一种开源通用策略引擎，可在整个堆栈中实现统一的、上下文感知的策略实施。该项目于2018年4月被CNCF沙箱接受，2021年2月4日正式毕业于CNCF。来自大约 30 个组织的 90 多人为 OPA 做出了贡献，维护者来自包括 Google、Microsoft、VMware 和 Styra。

简单来说，是在服务上抽象一层，统一控制、审计，本文讨论仅限在Kubernetes中的gatekeeper，对容器创建进行安全约束，确保符合运维规范。

1. 安装过程略 https://www.openpolicyagent.org/docs/latest/kubernetes-introduction/
2. 文件结构，规则、范围一一对应。例：default命名空间必须设置探针，规则名 k8srequiredprobes.yaml ，应用范围名 default_ns_must_have_probes.yaml

# k8srequiredprobes.yaml
apiVersion: templates.gatekeeper.sh/v1beta1
kind: ConstraintTemplate
metadata:
  name: k8srequiredprobes
spec:
  crd:
    spec:
      names:
        kind: K8sRequiredProbes
      validation:
        openAPIV3Schema:
          properties:
            message:
              type: string
  targets:
    - target: admission.k8s.gatekeeper.sh
      rego: |
        package k8srequiredprobes

        violation[{"msg": msg}] {
          spec := input.review.object.spec
          containers := spec.template.spec.containers
          some i
          not containers[i].readinessProbe
          msg := sprintf("container <%v> has no readiness probe", [containers[i].name])
        }

        violation[{"msg": msg}] {
          spec := input.review.object.spec
          containers := spec.template.spec.containers
          some i
          not containers[i].livenessProbe
          msg := sprintf("container <%v> has no liveness probe", [containers[i].name])
        }

# default_ns_must_have_probes.yaml
apiVersion: constraints.gatekeeper.sh/v1beta1
kind: K8sRequiredProbes
metadata:
  name: require-probes
spec:
  match:
    kinds:
      - apiGroups: ["apps"]
        kinds: ["Deployment"]
    namespaces:
      - "default"
  parameters:
    message: "A readiness and liveness probe is required"

# create_nginx.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-deployment
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - image: nginx:1.14.2
        name: nginx
        ports:
        - containerPort: 80

1. 应用规则、查看

kubectl apply -f k8srequiredprobes.yaml
kubectl apply -f default_ns_must_have_probes.yaml
# 执行创建nginx，可看到违规报错
kubectl apply -f create_nginx.yaml

# 查看违规数量（结果有延迟，10-15秒）
kubectl get constaints
# 查看违规详情（结果有延迟，10-15秒）
kubectl get constaints -o yaml

1. 复杂案例，对探针、副本数、资源限制范围、设计排除

# k8srequiredresourcelimitinrange.yaml 
apiVersion: templates.gatekeeper.sh/v1beta1
kind: ConstraintTemplate
metadata:
  name: k8srequiredresources
spec:
  crd:
    spec:
      names:
        kind: K8sRequiredResources
      validation:
        openAPIV3Schema:
          properties:
            message:
              type: string
  targets:
    - target: admission.k8s.gatekeeper.sh
      rego: |
        package k8srequiredresources

        violation[{"msg": msg}] {
          spec := input.review.object.spec
          containers := spec.template.spec.containers
          some i
          not containers[i].resources.requests
          msg := sprintf("container <%v> has no resource requests", [containers[i].name])
        }

        violation[{"msg": msg}] {
          spec := input.review.object.spec
          containers := spec.template.spec.containers
          some i
          not containers[i].resources.limits
          msg := sprintf("container <%v> has no resource limits", [containers[i].name])
        }

        violation[{"msg": msg}] {
          spec := input.review.object.spec
          containers := spec.template.spec.containers
          some i
          cpu := containers[i].resources.requests.cpu
          not cpu_within_range(cpu)
          msg := sprintf("container <%v> CPU request <%v> not within range", [containers[i].name, cpu])
        }

        violation[{"msg": msg}] {
          spec := input.review.object.spec
          containers := spec.template.spec.containers
          some i
          cpu := containers[i].resources.limits.cpu
          not cpu_within_range(cpu)
          msg := sprintf("container <%v> CPU limit <%v> not within range", [containers[i].name, cpu])
        }

        violation[{"msg": msg}] {
          spec := input.review.object.spec
          containers := spec.template.spec.containers
          some i
          memory := containers[i].resources.requests.memory
          not memory_within_range(memory)
          msg := sprintf("container <%v> memory request <%v> not within range", [containers[i].name, memory])
        }

        violation[{"msg": msg}] {
          spec := input.review.object.spec
          containers := spec.template.spec.containers
          some i
          memory := containers[i].resources.limits.memory
          not memory_within_range(memory)
          msg := sprintf("container <%v> memory limit <%v> not within range", [containers[i].name, memory])
        }

        cpu_within_range(cpu) {
          cpu_float := to_number(cpu)
          cpu_float >= 0.1
          cpu_float <= 1
        }

        memory_within_range(memory) {
          memory_int := to_number(memory)
          memory_int >= 50
          memory_int <= 100
        }


# k8srequiredatleast2replicas.yaml
apiVersion: templates.gatekeeper.sh/v1beta1
kind: ConstraintTemplate
metadata:
  name: k8sminreplicas
spec:
  crd:
    spec:
      names:
        kind: K8sMinReplicas
      validation:
        openAPIV3Schema:
          properties:
            minReplicas:
              type: integer
  targets:
    - target: admission.k8s.gatekeeper.sh
      rego: |
        package k8sminreplicas

        violation[{"msg": msg}] {
          spec := input.review.object.spec
          replicas := spec.replicas
          replicas <= input.parameters.minReplicas
          msg := sprintf("The number of replicas <%v> is less than or equal to the minimum <%v>", [replicas, input.parameters.minReplicas])
        }

# all_ns_must_have_resource_limit_in_range.yaml
apiVersion: constraints.gatekeeper.sh/v1beta1
kind: K8sRequiredResources
metadata:
  name: require-resources
spec:
  match:
    kinds:
      - apiGroups: ["apps"]
        kinds: ["Deployment"]
    excludedNamespaces: ["kube-system", "calico-system", "gatekeeper-system", "tigera-operator", "calico-apiserver"]
    labelSelector:
      matchExpressions:
      - {key: "gatekeeper-ignore", operator: NotIn, values: ["true"]}
  parameters:
    message: "Resource requests and limits are required and must be within range"

# all_ns_must_have_2_replicas.yaml
apiVersion: constraints.gatekeeper.sh/v1beta1
kind: K8sMinReplicas
metadata:
  name: min-replicas
spec:
  match:
    kinds:
      - apiGroups: ["apps"]
        kinds: ["Deployment"]
    excludedNamespaces: ["kube-system", "calico-system", "gatekeeper-system", "tigera-operator"]
    labelSelector:
      matchExpressions:
      - {key: "gatekeeper-ignore", operator: NotIn, values: ["true"]}
  parameters:
    minReplicas: 1

1. 添加例外，排除通过 {key: "gatekeeper-ignore", operator: NotIn, values: ["true"]} 实现，添加 gatekeeper-ignore: "true" 即可

# create_nginx_gatekeeper_ignore.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-deployment
  labels:
    gatekeeper-ignore: "true"
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - image: nginx:1.14.2
        name: nginx
        ports:
        - containerPort: 80