manifests-all.yaml

# Derived from ./manifests
---
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: elasticsearch
  labels:
    app: elasticsearch
    stack: logging
rules:
- apiGroups:
  - extensions
  resources:
  - podsecuritypolicies
  resourceNames:
  - elasticsearch
  verbs:
  - use
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: elasticsearch
  labels:
    app: elasticsearch
    stack: logging
subjects:
- kind: ServiceAccount
  name: elasticsearch
  namespace: default
roleRef:
  kind: ClusterRole
  name: elasticsearch
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: elasticsearch
  labels:
    app: elasticsearch
    stack: logging
---
apiVersion: extensions/v1beta1
kind: PodSecurityPolicy
metadata:
  name: elasticsearch
  labels:
    app: elasticsearch
    stack: logging
spec:
  fsGroup:
    rule: RunAsAny
  privileged: true
  runAsUser:
    rule: RunAsAny
  seLinux:
    rule: RunAsAny
  allowedCapabilities:
  - 'IPC_LOCK'
  - 'SYS_RESOURCE'
  supplementalGroups:
    rule: RunAsAny
  volumes:
  - '*'
  hostPID: true
  hostIPC: true
  hostNetwork: true
  hostPorts:
  - min: 1
    max: 65536
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: elasticsearch
  labels:
    app: elasticsearch
    stack: logging
data:
  elasticsearch.yml: |
    cluster.name: full-stack-cluster
    node.name: node-1
    path.data: /usr/share/elasticsearch/data
    http:
      host: 0.0.0.0
      port: 9200
    bootstrap.memory_lock: true
    transport.host: 127.0.0.1
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: elasticsearch
  labels:
    app: elasticsearch
    stack: logging
spec:
  replicas: 1
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: elasticsearch
        stack: logging
    spec:
      serviceAccountName: elasticsearch
      initContainers:
      - name: set-vm-max-map-count
        image: busybox
        imagePullPolicy: IfNotPresent
        command: ['sysctl', '-w', 'vm.max_map_count=262144']
        securityContext:
          privileged: true
      - name: volume-mount-hack
        image: busybox
        imagePullPolicy: IfNotPresent
        command: ["sh", "-c", "chown -R 1000:100 /usr/share/elasticsearch/data"]
        volumeMounts:
        - name: data
          mountPath: /usr/share/elasticsearch/data
      containers:
      - name: elasticsearch
        image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.0.0
        imagePullPolicy: IfNotPresent
        env:
        - name: ES_JAVA_OPTS
          value: -Xms1024m -Xmx1024m
          # ES_MEM_LIMIT=2g
          # ES_JVM_HEAP=1024m
        ports:
        - containerPort: 9200
        resources:
          limits:
            memory: "2147483648"
        volumeMounts:
        - name: config
          mountPath: /usr/share/elasticsearch/elasticsearch.yml
          subPath: elasticsearch.yml
        - name: data
          mountPath: /usr/share/elasticsearch/data
      # Allow non-root user to access PersistentVolume
      securityContext:
        fsGroup: 1000
      restartPolicy: Always
      volumes:
      - name: config
        configMap:
          name: elasticsearch
      - name: data
        persistentVolumeClaim:
          claimName: elasticsearch
      # - name: data
      #   hostPath:
      #     path: /srv/elasticsearch-data
---
# apiVersion: extensions/v1beta1
# kind: Ingress
# metadata:
#   name: elasticsearch
#   labels:
#     app: elasticsearch
#     stack: logging
# spec:
#   rules:
#   -
#     # host: elasticsearch.minikube.localnet
#     host: <YOUR_HOST>
#     http:
#       paths:
#       - path: /
#         backend:
#           serviceName: elasticsearch
#           servicePort: 9200
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: elasticsearch
  labels:
    app: elasticsearch
    stack: logging
spec:
  # storageClassName: standard
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 1G
---
apiVersion: v1
kind: Service
metadata:
  name: elasticsearch
  labels:
    app: elasticsearch
    stack: logging
spec:
  # needed on minikube
  type: NodePort
  ports:
  - name: "api"
    port: 9200
    targetPort: 9200
  selector:
    app: elasticsearch
    stack: logging
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: fluentd
  namespace: default
  labels:
    app: fluentd
    stack: logging
rules:
- apiGroups:
  - extensions
  resources:
  - podsecuritypolicies
  resourceNames:
  - fluentd
  verbs:
  - use
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: fluentd
  namespace: default
  labels:
    app: fluentd
    stack: logging
subjects:
- kind: ServiceAccount
  name: fluentd
roleRef:
  kind: Role
  name: fluentd
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: fluentd
  labels:
    app: fluentd
    stack: logging
---
apiVersion: extensions/v1beta1
kind: PodSecurityPolicy
metadata:
  name: fluentd
  labels:
    app: fluentd
    stack: logging
spec:
  fsGroup:
    rule: RunAsAny
  runAsUser:
    rule: RunAsAny
  seLinux:
    rule: RunAsAny
  supplementalGroups:
    rule: RunAsAny
  volumes:
  - emptyDir
  - secret
  - downwardAPI
  - configMap
  - persistentVolumeClaim
  - projected
  - hostPath
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: fluentd
  labels:
    app: fluentd
    stack: logging
rules:
- apiGroups: [""] # core API group
  resources: ["pods", "namespaces"]
  verbs: ["get", "watch", "list"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: fluentd
  labels:
    app: fluentd
    stack: logging
subjects:
- kind: ServiceAccount
  name: fluentd
  namespace: default
roleRef:
  kind: ClusterRole
  name: fluentd
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: fluentd
  # namespace: logging-fluentd
  labels:
    app: fluentd
data:
  output.conf: |
    # Enriches records with Kubernetes metadata
    <filter kubernetes.**>
      @type kubernetes_metadata
    </filter>
    
    <match **>
      @id elasticsearch
      @type elasticsearch
      @log_level info
      include_tag_key true
      type_name fluentd
      host "#{ENV['OUTPUT_HOST']}"
      port "#{ENV['OUTPUT_PORT']}"
      logstash_format true
      <buffer>
        @type file
        path /var/log/fluentd-buffers/kubernetes.system.buffer
        flush_mode interval
        retry_type exponential_backoff
        flush_thread_count 2
        flush_interval 5s
        retry_forever
        retry_max_interval 30
        chunk_limit_size "#{ENV['OUTPUT_BUFFER_CHUNK_LIMIT']}"
        queue_limit_length "#{ENV['OUTPUT_BUFFER_QUEUE_LIMIT']}"
        overflow_action block
      </buffer>
    </match>
  

  kubernetes.conf: |
    # This configuration file for Fluentd / td-agent is used
    # to watch changes to Docker log files. The kubelet creates symlinks that
    # capture the pod name, namespace, container name & Docker container ID
    # to the docker logs for pods in the /var/log/containers directory on the host.
    # If running this fluentd configuration in a Docker container, the /var/log
    # directory should be mounted in the container.
    #
    # These logs are then submitted to Elasticsearch which assumes the
    # installation of the fluent-plugin-elasticsearch & the
    # fluent-plugin-kubernetes_metadata_filter plugins.
    # See https://github.com/uken/fluent-plugin-elasticsearch &
    # https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter for
    # more information about the plugins.
    #
    # Example
    # =======
    # A line in the Docker log file might look like this JSON:
    #
    # {"log":"2014/09/25 21:15:03 Got request with path wombat\n",
    #  "stream":"stderr",
    #   "time":"2014-09-25T21:15:03.499185026Z"}
    #
    # The time_format specification below makes sure we properly
    # parse the time format produced by Docker. This will be
    # submitted to Elasticsearch and should appear like:
    # $ curl 'http://elasticsearch-logging:9200/_search?pretty'
    # ...
    # {
    #      "_index" : "logstash-2014.09.25",
    #      "_type" : "fluentd",
    #      "_id" : "VBrbor2QTuGpsQyTCdfzqA",
    #      "_score" : 1.0,
    #      "_source":{"log":"2014/09/25 22:45:50 Got request with path wombat\n",
    #                 "stream":"stderr","tag":"docker.container.all",
    #                 "@timestamp":"2014-09-25T22:45:50+00:00"}
    #    },
    # ...
    #
    # The Kubernetes fluentd plugin is used to write the Kubernetes metadata to the log
    # record & add labels to the log record if properly configured. This enables users
    # to filter & search logs on any metadata.
    # For example a Docker container's logs might be in the directory:
    #
    #  /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b
    #
    # and in the file:
    #
    #  997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log
    #
    # where 997599971ee6... is the Docker ID of the running container.
    # The Kubernetes kubelet makes a symbolic link to this file on the host machine
    # in the /var/log/containers directory which includes the pod name and the Kubernetes
    # container name:
    #
    #    synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
    #    ->
    #    /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log
    #
    # The /var/log directory on the host is mapped to the /var/log directory in the container
    # running this instance of Fluentd and we end up collecting the file:
    #
    #   /var/log/containers/synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
    #
    # This results in the tag:
    #
    #  var.log.containers.synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
    #
    # The Kubernetes fluentd plugin is used to extract the namespace, pod name & container name
    # which are added to the log message as a kubernetes field object & the Docker container ID
    # is also added under the docker field object.
    # The final tag is:
    #
    #   kubernetes.var.log.containers.synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
    #
    # And the final log record look like:
    #
    # {
    #   "log":"2014/09/25 21:15:03 Got request with path wombat\n",
    #   "stream":"stderr",
    #   "time":"2014-09-25T21:15:03.499185026Z",
    #   "kubernetes": {
    #     "namespace": "default",
    #     "pod_name": "synthetic-logger-0.25lps-pod",
    #     "container_name": "synth-lgr"
    #   },
    #   "docker": {
    #     "container_id": "997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b"
    #   }
    # }
    #
    # This makes it easier for users to search for logs by pod name or by
    # the name of the Kubernetes container regardless of how many times the
    # Kubernetes pod has been restarted (resulting in a several Docker container IDs).
    # Json Log Example:
    # {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"}
    # CRI Log Example:
    # 2016-02-17T00:04:05.931087621Z stdout F [info:2016-02-16T16:04:05.930-08:00] Some log text here
    <source>
      @id fluentd-containers.log
      @type tail
      path /var/log/containers/*.log
      pos_file /var/log/fluentd-containers.log.pos
      time_format %Y-%m-%dT%H:%M:%S.%NZ
      tag raw.kubernetes.*
      format json
      read_from_head true
    </source>
    # Detect exceptions in the log output and forward them as one log entry.
    <match raw.kubernetes.**>
      @id raw.kubernetes
      @type detect_exceptions
      remove_tag_prefix raw
      message log
      stream stream
      multiline_flush_interval 5
      max_bytes 500000
      max_lines 1000
    </match>
    # Multi-line parsing is required for all the kube logs because very large log
    # statements, such as those that include entire object bodies, get split into
    # multiple lines by glog.
    # Example:
    # I0204 07:32:30.020537    3368 server.go:1048] POST /stats/container/: (13.972191ms) 200 [[Go-http-client/1.1] 10.244.1.3:40537]
    <source>
      @id kubelet.log
      @type tail
      format multiline
      multiline_flush_interval 5s
      format_firstline /^\w\d{4}/
      format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
      time_format %m%d %H:%M:%S.%N
      path /var/log/kubelet.log
      pos_file /var/log/kubelet.log.pos
      tag kubelet
    </source>
    # Example:
    # I1118 21:26:53.975789       6 proxier.go:1096] Port "nodePort for kube-system/default-http-backend:http" (:31429/tcp) was open before and is still needed
    <source>
      @id kube-proxy.log
      @type tail
      format multiline
      multiline_flush_interval 5s
      format_firstline /^\w\d{4}/
      format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
      time_format %m%d %H:%M:%S.%N
      path /var/log/kube-proxy.log
      pos_file /var/log/kube-proxy.log.pos
      tag kube-proxy
    </source>
    # Example:
    # I0204 07:00:19.604280       5 handlers.go:131] GET /api/v1/nodes: (1.624207ms) 200 [[kube-controller-manager/v1.1.3 (linux/amd64) kubernetes/6a81b50] 127.0.0.1:38266]
    <source>
      @id kube-apiserver.log
      @type tail
      format multiline
      multiline_flush_interval 5s
      format_firstline /^\w\d{4}/
      format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
      time_format %m%d %H:%M:%S.%N
      path /var/log/kube-apiserver.log
      pos_file /var/log/kube-apiserver.log.pos
      tag kube-apiserver
    </source>
    # Example:
    # I0204 06:55:31.872680       5 servicecontroller.go:277] LB already exists and doesn't need update for service kube-system/kube-ui
    <source>
      @id kube-controller-manager.log
      @type tail
      format multiline
      multiline_flush_interval 5s
      format_firstline /^\w\d{4}/
      format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
      time_format %m%d %H:%M:%S.%N
      path /var/log/kube-controller-manager.log
      pos_file /var/log/kube-controller-manager.log.pos
      tag kube-controller-manager
    </source>
    # Example:
    # W0204 06:49:18.239674       7 reflector.go:245] pkg/scheduler/factory/factory.go:193: watch of *api.Service ended with: 401: The event in requested index is outdated and cleared (the requested history has been cleared [2578313/2577886]) [2579312]
    <source>
      @id kube-scheduler.log
      @type tail
      format multiline
      multiline_flush_interval 5s
      format_firstline /^\w\d{4}/
      format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
      time_format %m%d %H:%M:%S.%N
      path /var/log/kube-scheduler.log
      pos_file /var/log/kube-scheduler.log.pos
      tag kube-scheduler
    </source>

  # prometheus.conf: |
  #   # https://github.com/kazegusuri/fluent-plugin-prometheus#usage
  #
  #   <source>
  #     @type prometheus
  #   </source>
  #
  #   <source>
  #     @type prometheus_monitor
  #   </source>
  #
  #   <filter **>
  #     @type prometheus
  #     <metric>
  #       name fluentd_records_total
  #       type counter
  #       desc The total number of records read by fluentd.
  #     </metric>
  #   </filter>
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: fluentd
  labels:
    app: fluentd
spec:
  template:
    metadata:
      name: fluentd
      labels:
        app: fluentd
    spec:
      serviceAccountName: fluentd
      containers:
      - name: fluentd
        image: gcr.io/google-containers/fluentd-elasticsearch:v2.3.1
        imagePullPolicy: IfNotPresent
        env:
        - name: OUTPUT_HOST
          value: 'elasticsearch.default.svc'
        - name: OUTPUT_PORT
          value: '9200'
        - name: OUTPUT_BUFFER_CHUNK_LIMIT
          value: '2M'
        - name: OUTPUT_BUFFER_QUEUE_LIMIT
          value: '8'
        - name: K8S_NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        volumeMounts:
        - name: varlog
          mountPath: /var/log
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
        - name: libsystemddir
          mountPath: /host/lib
          readOnly: true
        - name: config-volume-fluentd
          mountPath: /etc/fluent/config.d
        # Liveness probe is aimed to help in situarions where fluentd
        # silently hangs for no apparent reasons until manual restart.
        # The idea of this probe is that if fluentd is not queueing or
        # flushing chunks for 5 minutes, something is not right. If
        # you want to change the fluentd configuration, reducing amount of
        # logs fluentd collects, consider changing the threshold or turning
        # liveness probe off completely.
        livenessProbe:
          initialDelaySeconds: 600
          periodSeconds: 60
          exec:
            command:
            - '/bin/sh'
            - '-c'
            - >
              LIVENESS_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-300};
              STUCK_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-900};
              if [ ! -e /var/log/fluentd-buffers ];
              then
                exit 1;
              fi;
              touch -d "${STUCK_THRESHOLD_SECONDS} seconds ago" /tmp/marker-stuck;
              if [[ -z "$(find /var/log/fluentd-buffers -type f -newer /tmp/marker-stuck -print -quit)" ]];
              then
                rm -rf /var/log/fluentd-buffers;
                exit 1;
              fi;
              touch -d "${LIVENESS_THRESHOLD_SECONDS} seconds ago" /tmp/marker-liveness;
              if [[ -z "$(find /var/log/fluentd-buffers -type f -newer /tmp/marker-liveness -print -quit)" ]];
              then
                exit 1;
              fi;
      volumes:
      - name: varlog
        hostPath:
          path: /var/log
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers
      # It is needed to copy systemd library to decompress journals
      - name: libsystemddir
        hostPath:
          path: /usr/lib64
      - name: config-volume-fluentd
        configMap:
          name: fluentd
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: kibana
  labels:
    app: kibana
    stack: logging
data:
  kibana.yml: |
    server:
      name: "full-stack-example"
      port: 127.0.0.1:5601
    elasticsearch.url: "http://elasticsearch:9200"
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: kibana
  labels:
    app: kibana
    stack: logging
spec:
  replicas: 1
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: kibana
        stack: logging
    spec:
      # FIXME
      # healthcheck + resources
      containers:
      - name: kibana
        image: docker.elastic.co/kibana/kibana-oss:6.0.0
        imagePullPolicy: IfNotPresent
        env:
        - name: ELASTICSEARCH_PASSWORD
          value: changeme
        ports:
        - containerPort: 5601
        resources: {}
        volumeMounts:
        - name: config
          mountPath: /usr/share/kibana/kibana.yml
          subPath: kibana.yml
      restartPolicy: Always
      volumes:
      - name: config
        configMap:
          name: kibana
---
# apiVersion: extensions/v1beta1
# kind: Ingress
# metadata:
#   name: kibana
#   labels:
#     app: kibana
#     stack: logging
#   annotations:
#     ingress.kubernetes.io/auth-signin: https://$host/oauth2/start
#     ingress.kubernetes.io/auth-url: https://$host/oauth2/auth
#     kubernetes.io/tls-acme: "true"
# spec:
#   rules:
#   - host: <YOUR_HOST>
#     http:
#       paths:
#       - path: /
#         backend:
#           serviceName: kibana
#           servicePort: 5601
#   tls:
#   - secretName: kibana-tls
#     hosts:
#     - <YOUR_HOST>

# ---
# apiVersion: extensions/v1beta1
# kind: Ingress
# metadata:
#   name: kibana-auth
#   labels:
#     app: kibana-auth
#     stack: logging
#   annotations:
#     kubernetes.io/tls-acme: "true"
# spec:
#   rules:
#   - host: <YOUR_HOST>
#     http:
#       paths:
#       - path: /oauth2
#         backend:
#           serviceName: oauth2-proxy
#           servicePort: 4180
#   tls:
#   - secretName: kibana-tls
#     hosts:
#     - <YOUR_HOST>
---
apiVersion: v1
kind: Service
metadata:
  name: kibana
  labels:
    app: kibana
    stack: logging
spec:
  type: NodePort
  ports:
  - name: "ui"
    port: 5601
    targetPort: 5601
  selector:
    app: kibana
    stack: logging
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: oauth2-proxy
  labels:
    app: oauth2-proxy
data:
  oauth2_proxy.cfg: |
    # for reference see https://github.com/bitly/oauth2_proxy/blob/master/contrib/oauth2_proxy.cfg.example

    http_address = "0.0.0.0:4180"
    upstreams = ["file:///dev/null"]

    provider = "github"
    email_domains = ["*"]
    github_org = "<GITHUB_ORG>"
    github_team = "<GITHUB_TEAM>"
    client_id = "<OIDC_ID>"
    client_secret = "<OIDC_SECRET>"
    cookie_secret = "<OIDC_COOKIE>"
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: oauth2-proxy
  labels:
    app: oauth2-proxy
spec:
  replicas: 1
  selector:
    matchLabels:
      app: oauth2-proxy
  template:
    metadata:
      labels:
        app: oauth2-proxy
    spec:
      containers:
      - name: oauth2-proxy
        image: giantswarm/oauth2_proxy:master
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 4180
          protocol: TCP
        volumeMounts:
        - name: config
          mountPath: /etc/oauth2_proxy
      volumes:
      - name: config
        configMap:
          name: oauth2-proxy
---
apiVersion: v1
kind: Service
metadata:
  name: oauth2-proxy
  labels:
    app: oauth2-proxy
spec:
  ports:
  - name: http
    port: 4180
    protocol: TCP
    targetPort: 4180
  selector:
    app: oauth2-proxy