Fluentd — Kubernetes (Graylog, GELF)

За основу возьмем официальный Docker образ fluntd

Установим необходимые gem'ы для GELF протокола и Kubernetes'а, а так же укажем, что fluentd будет запускаться от пользователя root, иначе будет получать ошибку доступа.

Dockerfile:

FROM fluent/fluentd:latest

USER root
WORKDIR /home/fluent

RUN set -ex \
    && apk add --no-cache --virtual .build-deps \
        build-base \
        ruby-dev \
    && echo 'gem: --no-document' >> /etc/gemrc \
    && gem install fluent-plugin-secure-forward \
    && gem install fluent-plugin-record-reformer \
    && gem install fluent-plugin-gelf-hs \
    && gem install fluent-plugin-kubernetes_metadata_filter \
    && gem install fluent-plugin-rewrite-tag-filter \
    && apk del .build-deps \
    && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem

# Copy plugins
COPY plugins /fluentd/plugins/

# Environment variables
ENV FLUENTD_OPT=""
ENV FLUENTD_CONF="fluent.conf"

# For Kubernetes. Start Fluentd by user root
ENV FLUENT_UID="0"

# Run Fluentd
CMD fluentd -c /fluentd/etc/$FLUENTD_CONF -p /fluentd/plugins $FLUENTD_OPT

plugins/parser_kubernetes.rb:

#
# Fluentd
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
#

# The following Fluentd parser plugin, aims to simplify the parsing of multiline
# logs found in Kubernetes nodes. Since many log files shared the same format and
# in order to simplify the configuration, this plugin provides a 'kubernetes' format
# parser (built on top of MultilineParser).
#
# When tailing files, this 'kubernetes' format should be applied to the following
# log file sources:
#
#  - /var/log/kubelet.log
#  - /var/log/kube-proxy.log
#  - /var/log/kube-apiserver.log
#  - /var/log/kube-controller-manager.log
#  - /var/log/kube-scheduler.log
#  - /var/log/rescheduler.log
#  - /var/log/glbc.log
#  - /var/log/cluster-autoscaler.log
#
# Usage:
#
# ---- fluentd.conf ----
#
# <source>
#    type tail
#    format kubernetes
#    path ./kubelet.log
#    read_from_head yes
#    tag kubelet
# </source>
#
# ----   EOF       ---

require 'fluent/parser'

module Fluent
  class KubernetesParser < Fluent::TextParser::MultilineParser
    Fluent::Plugin.register_parser("kubernetes", self)

    CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/}
    CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/}
    CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N"

    def configure(conf)
      conf['format_firstline'] = CONF_FORMAT_FIRSTLINE
      conf['format1'] = CONF_FORMAT1
      conf['time_format'] = CONF_TIME_FORMAT
      super
    end
  end
end

После сборки кастомного образа, переходим к настройке Kubernetes

rbac.yaml:

apiVersion: v1
kind: ServiceAccount
metadata:
  name: kube-gelf
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  annotations:
    rbac.authorization.kubernetes.io/autoupdate: "true"
  name: kube-gelf-metadata-filter
rules:
- apiGroups:
  - ""
  resources:
  - pods
  - pods/exec
  - namespaces
  verbs: ["get", "list", "watch", "create"]
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: kube-gelf
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: kube-gelf-metadata-filter
subjects:
- kind: ServiceAccount
  name: kube-gelf
  namespace: kube-system

configmap.yml:

apiVersion: v1
kind: ConfigMap
metadata:
  name: kube-gelf
  namespace: kube-system
  labels:
    app: kube-gelf
data:
  GELF_HOST: "graylog-svc.staging.svc.cluster.local"
  GELF_PORT: "12201"
  GELF_PROTOCOL: "udp"
  fluent.conf: |
    <match fluent.**>
      @type null
    </match>

    <source>
      @type tail
      path /var/log/containers/*.log
      pos_file /pos/containers.pos
      time_key time
      time_format %Y-%m-%dT%H:%M:%S.%NZ
      tag kubernetes.*
      format json
      read_from_head true
    </source>

    <filter kubernetes.**>
      @type kubernetes_metadata
      ca_file /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file /var/run/secrets/kubernetes.io/serviceaccount/token
    </filter>

    <filter access>
      @type record_transformer
      <record>
        hostname "#{ENV['NODENAME']}"
      </record>
    </filter>

    ### NOT COLLECT LOGS FROM ALL EXCEPT NAMESPACE STAGING ###

    <match **fluentd**>
      @type null
    </match>

    <match **kube-system**>
      @type null
    </match>

    <match **monitoring**>
      @type null
    </match>

    <match **cert-manager**>
      @type null
    </match>

    <match **efs-staging**>
      @type null
    </match>

    ##########################################################

    <match kubernetes.**>
      @type rewrite_tag_filter
      <rule>
        key $['kubernetes']['labels']['app']
        pattern ^(.+)$
        tag $1
      </rule>
    </match>

    <match **>
       @type copy
       <store>
         @type gelf
         include_tag_key true
         host "#{ENV['GELF_HOST']}"
         port "#{ENV['GELF_PORT']}"
         protocol "#{ENV['GELF_PROTOCOL']}"
         flush_interval 10s
         num_threads 2
         use_record_host true
         buffer_chunk_limit 4096K
         buffer_queue_limit 512
         max_retry_wait 300
       </store>
    </match>

    # Valid log_level's are: fatal, error, warn, info, debug, trace
    <system>
      log_level warn
    </system>

В файле "fluent.conf" я исключил все, кроме namespace staging, в котором у меня крутятся все контейнеры.

manifest.yml:

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-gelf
  namespace: kube-system
spec:
  updateStrategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
  selector:
    matchLabels:
      app: kube-gelf
  template:
    metadata:
      labels:
        app: kube-gelf
    spec:
      serviceAccountName: kube-gelf
      dnsPolicy: ClusterFirst
      containers:
      - name: agent
        image: my-image-repo:kube-gelf
        imagePullPolicy: "IfNotPresent"
        env:
        - name: GELF_HOST
          valueFrom:
            configMapKeyRef:
              name: kube-gelf
              key: GELF_HOST
        - name: GELF_PORT
          valueFrom:
            configMapKeyRef:
              name: kube-gelf
              key: GELF_PORT
        - name: GELF_PROTOCOL
          valueFrom:
            configMapKeyRef:
              name: kube-gelf
              key: GELF_PROTOCOL
        - name: NODENAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        volumeMounts:
        - name: varlog
          mountPath: /var/log
          readOnly: true
        - name: gelf-pos
          mountPath: /pos
          readOnly: false
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
        - name: kube-gelf-config
          mountPath: /fluentd/etc/fluent.conf
          subPath: fluent.conf
      tolerations:
      - key: node-role.kubernetes.io/master
        operator: Exists
        effect: NoSchedule
      volumes:
      - name: varlog
        hostPath:
          path: /var/log
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers
      - name: kube-gelf-config
        configMap:
          name: kube-gelf
          items:
          - key: fluent.conf
            path: fluent.conf
      - name: gelf-pos
        hostPath:
          path: /var/log/pos

Не забываем указать свой путь с собраному образу.

0 0 vote
Рейтинг статьи

Метки: Метки

Подписаться
Уведомление о
guest
0 комментариев
Inline Feedbacks
View all comments