ÈÕÖ¾ÊÕ¼¯·½°¸EFK

data:   
    td-agent.conf:|-

<match fluent.**>
  @type null
</match>
<source>
  @id xx-containers.log
  @type tail
  path /var/log/**/log.log
  pos_file /var/log/xx.log.pos
  tag log.**
  <parse>
        @type multiline
        format_firstline /^\[\d{4}\/\d{2}\/\d{2} /
        format1 /^\[(?<logtime>[^\]]*)\] \[(?<level>[^ ]*)\] (?<position>[\s\S\d]*): \[Action=(?<action>[^ ]*)\|RequestId=(?<reqId>[^|]*)\|AppId=(?<appid>[^ ]*)\|TfAid=(?<aid>[^|]*)\|TfUid=(?<uid>[^|]*)\|(?<context>[^\]]*)\] (?<message>[\s\S]*)$/
  </parse>
  emit_unmatched_lines false
  read_from_head false
</source>
<source>
  @id xx-containers.log
  @type tail
  path /var/log/**/vlog.log
  pos_file /var/log/xx.log.pos
  tag xx.*
  <parse>
    @type regexp
    expression /(?<logtime>\d{4}-[01]\d-[0-3]\d [0-2]\d:[0-5]\d:[0-5]\d\.\d{3})[\t ]*(?<level>[^\t ]*)[\t ]*(?<position>[^\t ]*)[\t ]*(?<message>[\s\S]*)$/
  </parse>
  emit_unmatched_lines false
  read_from_head false
</source>
<filter **>
  @type record_transformer
  enable_ruby true
  <record>
    message ${record.dig("position")}:${record.dig("message")}
  </record>
</filter>
<filter **>
  @type record_transformer
  <record>
    namespace ${tag_parts[4]}
  </record>
</filter>
<filter **>
  @type record_transformer
  <record>
    module ${tag_parts[5]}
  </record>
</filter>
<filter **>
  @type record_transformer
  <record>
    service_name ${tag_parts[4]}:${tag_parts[5]}
  </record>
</filter>
<match **>
   @type elasticsearch_dynamic
   @log_level debug
   include_tag_key true
   type_name _doc
   host xx
   port xx
   user xx
   password xx
   logstash_format true
   logstash_prefix logstash-xx-test
   <buffer>
        @type file
        path /var/log/fluentd/buffer
        flush_interval 5s
        flush_thread_count 10
        retry_forever false
        retry_max_times 1
		retry_wait 1s
        chunk_limit_size 8MB
        total_limit_size 256MB
        queue_limit_length 32
        compress gzip
   </buffer>
</match>

as the final mapping would have more than 1 type: [_doc, fluentd]

apiVersion: apps/v1beta2
kind: DaemonSet
metadata:
  annotations:
    deprecated.daemonset.template.generation: "6"
    description: fluentdÈÕÖ¾ÊÕ¼¯×é¼þ
  generation: 7
  labels:
    k8s-app: fluentd-es
    qcloud-app: fluentd-es
  name: fluentd-es
  namespace: kube-public
spec:
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      k8s-app: fluentd-es
      qcloud-app: fluentd-es
  template:
    metadata:
      creationTimestamp: null
      labels:
        k8s-app: fluentd-es
        qcloud-app: fluentd-es
    spec:
      containers:
      - image: ccr.ccs.tencentyun.com/k8s-comm/fluentd-elasticsearch:v2.5.2
        imagePullPolicy: IfNotPresent
        name: fluentd-es
        resources:
          limits:
            cpu: "1"
            memory: 2Gi
          requests:
            cpu: 500m
            memory: 256Mi
        securityContext:
          privileged: false
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /var/log
          name: varlog
        - mountPath: /var/lib/docker/containers
          name: varlibdockercontainers
        - mountPath: /etc/fluent/config.d
          name: config-volume
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      serviceAccount: efk
      serviceAccountName: efk
      terminationGracePeriodSeconds: 30
      volumes:
      - hostPath:
          path: /var/log
          type: DirectoryOrCreate
        name: varlog
      - hostPath:
          path: /var/lib/docker/containers
          type: DirectoryOrCreate
        name: varlibdockercontainers
      - configMap:
          defaultMode: 420
          name: td-agent-config
        name: config-volume
  updateStrategy:
    type: OnDelete