title: “Fluentd YAML Files”
date: 2020-11-27T08:38:15
slug: fluentd-yaml-files

apiVersion: v1
kind: ServiceAccount
metadata:
 name: fluentd
 namespace: elasticsearch-azure
---

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
 name: fluentd-role
 namespace: elastisearch-azure
rules:
 - apiGroups: [""]
 resources:
 - namespaces
 - pods
 - pods/logs
 verbs: ["get", "list", "watch"]
---

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
 name: fluentd-role-binding
roleRef:
 apiGroup: rbac.authorization.k8s.io
 kind: ClusterRole
 name: fluentd-role
subjects:
 - kind: ServiceAccount
 name: fluentd
 namespace: elasticsearch-azure
---

apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
 name: fluentd
 namespace: elasticsearch-azure
 labels:
 k8s-app: fluentd-logging
 version: v1
spec:
 template:
 metadata:
 labels:
 k8s-app: fluentd-logging
 version: v1
 spec:
 serviceAccountName: fluentd
 tolerations:
 - key: node-role.kubernetes.io/master
 effect: NoSchedule
 containers:
 - name: fluentd
 image: fluent/fluentd-kubernetes-daemonset:v1-debian-elasticsearch
 envFrom:
 - secretRef:
 name: fluent-tls
 env:
 - name: FLUENT\_ELASTICSEARCH\_HOST
 value: "{{server\_namne}}"
 - name: FLUENT\_ELASTICSEARCH\_PORT
 value: "{port}"
 - name: FLUENT\_ELASTICSEARCH\_SCHEME
 value: "https"
 # Option to configure elasticsearch plugin with self signed certs
 # ================================================================
 - name: FLUENT\_ELASTICSEARCH\_SSL\_VERIFY
 value: "true"
 # Option to configure elasticsearch plugin with tls
 # ================================================================
 - name: FLUENT\_ELASTICSEARCH\_SSL\_VERSION
 value: "TLSv1\_2"
 resources:
 limits:
 memory: 200Mi
 requests:
 cpu: 100m
 memory: 200Mi
 volumeMounts:
 - name: varlog
 mountPath: /var/log
 - name: varlibdockercontainers
 mountPath: /var/lib/docker/containers
 readOnly: true
 - name: ssl
 mountPath: /fluent-tls/ssl
 readOnly: true
 terminationGracePeriodSeconds: 30
 volumes:
 - name: varlog
 hostPath:
 path: /var/log
 - name: varlibdockercontainers
 hostPath:
 path: /var/lib/docker/containers
 # certificates folder for filebeat
 - name: ssl
 secret:
 secretName: fluent-tls

kubectl create secret generic fluent-tls \
--from-file=ca\_file=./chain.pem \
--from-file=cert\_pem=./cert.pem \
--from-file=cert\_key=./cert.key

title: “Read Kubernetes or Openshift Audit Logs, flag it and send it to ELS”
date: 2020-11-27T08:35:23
slug: read-kubernetes-or-openshift-audir-logs-flag-it-and-send-it-to-els

# Openshift audit logs

 @type tail
 @id openshift-audit-input
 path /var/log/oauth-apiserver/audit.log,/var/log/openshift-apiserver/audit.log
 pos\_file /tmp/audit.log.pos
 tag openshift-audit.log

 @type json
 time\_key requestReceivedTimestamp
 keep\_time\_key true
 time\_format %Y-%m-%dT%H:%M:%S.%N%z


 @type copy

 @type elasticsearch
# @id default
 @log\_level "info"
 include\_tag\_key true
 host "opendistro"
 port 9200
 scheme https
 ssl\_verify false
 ssl\_version TLSv1\_2
 client\_cert /etc/fluent/cert/cert\_pem
 client\_key /etc/fluent/cert/cert\_key
 client\_cert\_auth true
 reload\_connections false
 reconnect\_on\_error true
 reload\_on\_failure true
 log\_es\_400\_reason false
 logstash\_prefix "audit-openshift"
 logstash\_format true
 index\_name "audit-openshift"
 type\_name "fluentd"

 flush\_thread\_count 1
 flush\_interval 5s
 chunk\_limit\_size 2M
 queue\_limit\_length 4
 retry\_max\_interval 30
 retry\_forever true

title: “Read containerlogs, flag it and fromat it and send it to els”
date: 2020-11-27T08:34:26
slug: read-containerlogs-flag-it-and-fromat-it-and-send-it-to-els

# Container Logs

 @type tail
 path /var/log/containers/\*.log
 exclude\_path ["/var/log/containers/fluentd-\*\_openshift-logging\_\*.log"]
 pos\_file /tmp/containers.log.pos
 refresh\_interval 5
 rotate\_wait 5
 tag kubernetes.\*
 format json
 read\_from\_head true

 @type multi\_format

 format json
 time\_format '%Y-%m-%dT%H:%M:%S.%N%Z'
 keep\_time\_key true


 format regexp
 expression /^(?.+) (?stdout|stderr)( (?.))? (?.\*)$/
 time\_format '%Y-%m-%dT%H:%M:%S.%N%:z'
 keep\_time\_key true



 type kubernetes\_metadata

 @type copy

 @type elasticsearch
 @type elasticsearch\_dynamic
 # @id default
 @log\_level "info"
 include\_tag\_key true
 host "opendistro"
 port 9200
 scheme https
 ssl\_verify false
 ssl\_version TLSv1\_2
 client\_cert /etc/fluent/cert/cert\_pem
 client\_key /etc/fluent/cert/cert\_key
 client\_cert\_auth true
 reload\_connections false
 reconnect\_on\_error true
 reload\_on\_failure true
 log\_es\_400\_reason false
 #ogstash\_prefix "containers"
 logstash\_prefix logstash-${record['kubernetes']['namespace\_name']}
 logstash\_format true
 index\_name "containers"
 type\_name "fluentd"

 flush\_thread\_count 1
 flush\_interval 5s
 chunk\_limit\_size 2M
 queue\_limit\_length 4
 retry\_max\_interval 30
 retry\_forever true

title: “Indexname from Namespace Name”
date: 2020-11-27T08:33:22
slug: inexname-from-namespace-name

@type copy
@type elasticsearch
@type elasticsearch\_dynamic
# @id default
@log\_level "info"
include\_tag\_key true
host "opendistro"
port 9200
scheme https
ssl\_verify false
ssl\_version TLSv1\_2
client\_cert /etc/fluent/cert/cert\_pem
client\_key /etc/fluent/cert/cert\_key
client\_cert\_auth true
reload\_connections false
reconnect\_on\_error true
reload\_on\_failure true
log\_es\_400\_reason false
#ogstash\_prefix "containers"
logstash\_prefix logstash-${record['kubernetes']['namespace\_name']}
logstash\_format true
index\_name "containers"
type\_name "fluentd"
flush\_thread\_count 1
flush\_interval 5s
chunk\_limit\_size 2M
queue\_limit\_length 4
retry\_max\_interval 30
retry\_forever true

title: “Params from ENV”
date: 2020-11-27T08:32:24
slug: params-from-env

 @type elasticsearch\_dynamic
 #user "#{ENV['FLUENT\_ELASTICSEARCH\_USER']}"
 #password "#{ENV['FLUENT\_ELASTICSEARCH\_PASSWORD']}"
 @log\_level info
 include\_tag\_key true
 host "#{ENV['OUTPUT\_HOST']}"
 port "#{ENV['OUTPUT\_PORT']}"
 logstash\_format true
 logstash\_prefix logstash-${record['kubernetes']['namespace\_name']}

 @type file
 path /var/log/fluentd-buffers/kubernetes.system.buffer
 flush\_mode interval
 retry\_type exponential\_backoff
 flush\_thread\_count 2
 flush\_interval 5s
 retry\_forever
 retry\_max\_interval 30
 chunk\_limit\_size "#{ENV['OUTPUT\_BUFFER\_CHUNK\_LIMIT']}"
 queue\_limit\_length "#{ENV['OUTPUT\_BUFFER\_QUEUE\_LIMIT']}"
 overflow\_action block