浅谈微服务k8s情景下的日志采集

---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  namespace: kube-system
  name: filebeat
  labels:
    app: filebeat
spec:
  selector:
    matchLabels:
      app: filebeat
  template:
    metadata:
      labels:
        app: filebeat
    spec:
      serviceAccountName: filebeat
      terminationGracePeriodSeconds: 30
      hostNetwork: true
      dnsPolicy: ClusterFirstWithHostNet
      
      hostAliases:
      - ip: "192.168.201.126"
        hostnames:
        - "kafka01"

      
      containers:
      - name: filebeat
        image: docker.elastic.co/beats/filebeat:7.14.2
        args: [
          "-c", "/etc/filebeat.yml",
          "-e",
        ]
        env:
        - name: NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        securityContext:
          runAsUser: 0
        resources:
          limits:
            memory: 800Mi
          requests:
            cpu: 400m
            memory: 200Mi
        volumeMounts:
        - name: config
          mountPath: /etc/filebeat.yml
          readOnly: true
          subPath: filebeat.yml
        - name: data
          mountPath: /usr/share/filebeat/data
        - name: varlog
          mountPath: /var/log
          readOnly: true
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
        - name: dockersock
          mountPath: /var/run/docker.sock
      volumes:
      - name: config
        configMap:
          defaultMode: 0640
          name: filebeat-config
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers
      - name: dockersock
        hostPath:
          path: /var/run/docker.sock
      - name: varlog
        hostPath:
          path: /var/log
      - name: data
        hostPath:
          # When filebeat runs as non-root user, this directory needs to be writable by group (g+w).
          path: /var/lib/filebeat-data
          type: DirectoryOrCreate
---


# 权限配置  
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: filebeat
subjects:
- kind: ServiceAccount
  name: filebeat
  namespace: kube-system
roleRef:
  kind: ClusterRole
  name: filebeat
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: filebeat
  namespace: kube-system
subjects:
  - kind: ServiceAccount
    name: filebeat
    namespace: kube-system
roleRef:
  kind: Role
  name: filebeat
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: filebeat-kubeadm-config
  namespace: kube-system
subjects:
  - kind: ServiceAccount
    name: filebeat
    namespace: kube-system
roleRef:
  kind: Role
  name: filebeat-kubeadm-config
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: filebeat
  labels:
    app: filebeat
rules:
- apiGroups: [""] # "" indicates the core API group
  resources:
  - namespaces
  - pods
  - nodes
  verbs:
  - get
  - watch
  - list
- apiGroups: ["apps"]
  resources:
    - replicasets
  verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: filebeat
  # should be the namespace where filebeat is running
  namespace: kube-system
  labels:
    app: filebeat
rules:
  - apiGroups:
      - coordination.k8s.io
    resources:
      - leases
    verbs: ["get", "create", "update"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: filebeat-kubeadm-config
  namespace: kube-system
  labels:
    app: filebeat
rules:
  - apiGroups: [""]
    resources:
      - configmaps
    resourceNames:
      - kubeadm-config
    verbs: ["get"]
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: filebeat
  namespace: kube-system
  labels:
    app: filebeat
---

---
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: kube-system
  name: filebeat-config
  labels:
    app: filebeat
data:
  filebeat.yml: |-
    filebeat.autodiscover:
      providers:
        - type: kubernetes
          node: ${NODE_NAME}
          templates:
            - condition:
                equals:
                  kubernetes.labels.filebeat_harvest: "true"
              config:
                - type: container
                  encoding: utf-8
                  paths: 
                    - /var/log/containers/*${data.kubernetes.container.id}.log
                  # exclude_lines: ["^\\s+[\\-`('.|_]"]
                  # multiline:
                  #   max_lines: 10000
                  #   pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
                  #   negate: true
                  #   match: after
                  # symlinks: true
      # 过滤边车模式的filebeat日志
    processors:
      - drop_event.when:
          contains:
            - kubernetes.container.name: "filebeat"
    processors:
      - drop_fields:
          fields:
            - "@metadata"
            - "beat"
            - "kubernetes.labels"
            - "kubernetes.container"
            - "kubernetes.annotations"
            - "host"
            - "prospector"
            - "input"
            - "offset"
            - "stream"
            - "source"
            - "agent.ephemeral_id"
            - "agent.hostname"
            - "agent.id"
            - "agent.name"
            - "agent.type"
            - "agent.version"
            - "host.name"
            - "input.type"
            - "ecs.version"
            - "input.type"
            - "log.offset"
            - "log.flags"
            - "log.file.path"
            - "version"

    output.elasticsearch:
      hosts: ["http://172.18.145.131:9200"]
      enabled: true
      worker: 1
      compression_level: 3
      indices:
        - index: "%{[kubernetes.labels.filebeat_index]}-%{+yyyy.MM.dd}"
    setup.ilm.enabled: false
    setup.template.name: "ms"
    setup.template.pattern: "ms-*"
    setup.template.settings:
      index.number_of_shards: 1    
      index.number_of_replicas: 0  

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    tier: backend
  name: chat-api
  namespace: default
spec:
  replicas: 1
  selector:
    matchLabels:
      tier: backend
  template:
    metadata:
      labels:
        tier: backend
    spec:
      hostAliases:
      - ip: "192.168.201.1"
        hostnames:
        - "kafka01"
      nodeSelector:
        common: prod
      containers:
        - name: chat-api
          imagePullPolicy: Always
          ports:
            - containerPort: 8080
              protocol: TCP
          resources:
            limits:
              cpu: 1000m
              memory: 1024Mi
            requests:
              cpu: 1000m
              memory: 1024Mi
          volumeMounts:
            - name: service-logs-nas
              mountPath: /data/logs/chat
        - name: filebeat
          image: filebeat:6.4.2
          args: [
              "-c", "/etc/filebeat.yml",
              "-e",
          ]
          resources:
            requests:
              cpu: 10m
              memory: 30Mi
            limits:
              memory: 500Mi
          securityContext:
            runAsUser: 0
          volumeMounts:
            - name: filebeat-config
              mountPath: /etc/filebeat.yml
              subPath: filebeat.yml
            - name: service-logs-nas
              mountPath: /data/logs/chat
      volumes:
        # 这里因为有归档需求，所以通过NFS来共享存储
        # 如果没有归档需求，可以直接使用 emptyDir 来共享存储
        - name: service-logs-nas
          nfs:
            path: /prod-api
            server: xxxx.cn-shenzhen.nas.aliyuncs.com
        - name: filebeat-config
          configMap:
            name: filebeat-sidecar-chat-config
      dnsPolicy: ClusterFirst
      restartPolicy: Always

apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-sidecar-chat-config
  namespace: default
data:
  filebeat.yml: |-
    name: k8s-chat
    filebeat.shutdown_timeout: 3s

    filebeat.prospectors:
      - input_type: log
        paths:
          - /data/logs/chat/*${HOSTNAME}*.log
        json.keys_under_root: true
        json.add_error_key: true
        ignore_older: 12h
        close_removed: true
        clean_removed: true
        close_inactive: 2h
        fields:
          k8s_nodename: ${NODE_NAME}
          k8s_namespace: ${POD_OWN_NAMESPACE}
          type: k8s-chat
          format: json


    output.kafka:
        hosts: ["kafka01:20001]
        topic: '%{[fields.type]}'
        partition.round_robin:
            reachable_only: true
        username: "xxx"
        password: "xxx"
        required_acks: 1
        compression: gzip
        max_message_bytes: 1000000
        worker: 1