Skip to content

Deployment to K8S

Jump to bottom
Bryan Kok edited this page Mar 28, 2023 · 6 revisions

The details of the environment as deployed to https://deploy.plan.ovh are as follows:

Helm charts

CHART                   APP VERSION 
cert-manager-v1.9.1     v1.9.1      
ingress-nginx-4.2.3     1.3.0       
postgresql-11.1.28      14.2.0      
rabbitmq-9.0.7          3.9.17      
redis-16.9.3            6.2.7

Backend

application-prod.yaml 
spring:
  mail:
    debug: true
    host: 'CHANGE_THIS'
    port: 587
    protocol: smtp
    username: 'CHANGE_THIS'
    password: 'CHANGE_THIS'
    properties:
      "mail.transport.protocol": smtp
      "mail.smtp.auth": true
      "mail.tls": true
      "mail.smtp.starttls.enable": true
  autoconfigure:
    exclude:
      - "org.springframework.boot.autoconfigure.thymeleaf.ThymeleafAutoConfiguration"
  rabbitmq:
    host: rabbitmq-prod
    port: 5672
    username: CHANGE_THIS
    password: CHANGE_THIS
  redis:
    host: redis-prod-master
    port: 6379
    password: "CHANGE_THIS"
    flush-mode: immediate
  session:
    store-type: redis
    timeout: 2d
  flyway:
    enabled: true
    locations: "classpath:db/migration/{vendor}"
  # primary datasource
  datasource:
    username: "CHANGE_THIS"
    password: "CHANGE_THIS"
    url: "jdbc:postgresql://postgresql-prod:5432/CHANGE_THIS"

  security:
    oauth2:
      client:
        provider:
          github:
            authorization-uri: https://github.com/login/oauth/authorize
            token-uri: https://github.com/login/oauth/access_token
            user-info-uri: https://api.github.com/user
            user-name-attribute: name
          google:
            # needed to obtain the refresh token too
            # https://stackoverflow.com/questions/69146220/how-to-set-access-type-offline-using-spring-boot-security-and-oauth-flow-to-gene
            authorization-uri: "https://accounts.google.com/o/oauth2/v2/auth?access_type=offline&prompt=consent"
        registration:
          github:
            client-id: CHANGE_THIS
            client-secret: CHANGE_THIS
            provider: github
            redirect-uri: "{baseScheme}://{baseHost}{basePort}{basePath}/login/oauth2/code/github"
            scope:
              - email
              - profile
          google:
            # https://redirectmeto.com/{baseUrl}/login/oauth2/code/google
            redirect-uri: https://{baseHost}{basePort}{basePath}/login/oauth2/code/google 
            client-id: CHANGE_THIS.apps.googleusercontent.com
            client-secret: CHANGE_THIS
            scope:
              - email
              - profile
#              - openid

# comma-separated, see https://github.com/spring-projects/spring-framework/issues/16381
custom_cors:
        origins: https://deploy.plan.ovh

custom_app:
  rate_limit_key_prefix: "dapp_rlimit:"
  token_validity_duration: 3600
#  email_retry_cooldown: 600
  # two attempts in 5 minutes
  email_retry_rate_limit_duration: 5
  email_retry_rate_limit_capacity: 2
  email_from: CHANGE_THIS@CHANGE_THIS

management:
  health:
    probes:
      enabled: true
  info:
    git:
      mode: full
  endpoints:
    web:
      cors:
        allowed-origins: '*'
        allowed-methods: GET
      health:
        # https://stackoverflow.com/questions/46326527/spring-boot-actuator-endpoint-sensitivity-vs-security-enablement
        info:
          sensitive: false
      exposure:
        include:
          - info
          - beans
          - loggers
          - health

# https://springdoc.org/
# https://stackoverflow.com/questions/65820916/i-have-installed-openapi-3-using-springdoc-but-the-url-is-strange-can-i-change
springdoc:
  api-docs:
    path: /api-docs
  swagger-ui:
    path: /api-docs/ui

# for async processing with jobrunr
org:
  jobrunr:
    job-scheduler:
      enabled: true
    database:
      type: redis-lettuce
    dashboard:
      port: 8082
      enabled: true

logging:
  level:
    org:
      springframework:
        web: DEBUG
        security: DEBUG
#          filter:
#            CommonsRequestLoggingFilter: DEBUG

server:
  tomcat:
    threads:
      min-spare: 5
      max: 20
  forward-headers-strategy: framework # https://github.com/springdoc/springdoc-openapi/issues/171
  # port: 8081
  servlet:
    session:
      cookie:
        # https://stackoverflow.com/questions/33095345/how-to-change-spring-session-redis-cookie-name
        name: "DEPLOYAPP_SESSION"
        # needed when testing in staging
        same-site: None
        secure: true
        path: /
  error:
    include-message: always
    include-binding-errors: always
    include-stacktrace: on_param
    include-exception: true
kubectl create configmap deployapp-backend --from-file=./application-prod.yml
Deployment YAML 
kind: Deployment
apiVersion: apps/v1
metadata:
  name: deployapp-backend
spec:
  replicas: 1
  selector:
    matchLabels:
      app: deployapp-backend
  template:
    metadata:
      labels:
        app: deployapp-backend
    spec:
      containers:
      - name: deployapp-backend
        image: transfusion/deployapp-backend:latest
        ports:
        - containerPort: 8080
        imagePullPolicy: Always
        env:
        - name: SPRING_PROFILES_ACTIVE
          value: "prod"
        - name: SPRING_CONFIG_LOCATION
          value: "file:///config/"
        - name: JAVA_OPTS
          value: >-
                  -XX:MaxRAM=800M
        resources: {}
        livenessProbe:
          httpGet:
            path: /actuator/health/liveness
            port: 8080
          initialDelaySeconds: 120
        readinessProbe:
          httpGet:
            path: /actuator/health/readiness
            port: 8080
          initialDelaySeconds: 120
        lifecycle:
          preStop:
            exec:
              command: ["sh", "-c", "sleep 10"]
        volumeMounts:
            - name: config-volume
              mountPath: /config
      volumes:
        - name: config-volume
          configMap:
            name: deployapp-backend
status: {}
kubectl apply -f deployapp-backend.yaml
K8s service 
kind: Service
apiVersion: v1
metadata:
  name: deployapp-backend-service
spec:
  type: NodePort
  ports:
    - port: 8080
      targetPort: 8080
      protocol: TCP
  selector:
    app: deployapp-backend
kubectl apply -f deployapp-backend-service.yaml

Storage Service

application-prod.yaml 
spring:
  servlet:
    multipart:
      enabled: true
      max-file-size: 250MB
      max-request-size: 250MB
  rabbitmq:
    host: CHANGE_THIS
    port: 5672
    username: CHANGE_THIS
    password: CHANGE_THIS
  redis:
    host: redis-prod-master
    port: 6379
    password: "CHANGE_THIS"
    flush-mode: immediate
  session:
    store-type: redis
    timeout: 2d
  flyway:
    enabled: true
    locations: "classpath:db/migration/{vendor}"
  # primary datasource
  datasource:
    username: "CHANGE_THIS"
    password: "CHANGE_THIS"
    url: "jdbc:postgresql://postgresql-prod:5432/CHANGE_THIS"

# comma-separated, see https://github.com/spring-projects/spring-framework/issues/16381
custom_cors:
        origins: https://deploy.plan.ovh

custom_app:
  lock_registry_key_prefix: "dapp_storage_lock"
  # app parsing threads
  core_pool_size: 2
  max_pool_size: 2
  queue_capacity: 40 # 60 mins / 1.5 min per app parsing (conservative estimate)
  rubyCleanSleep: 5000 

management:
  health:
    probes:
      enabled: true
  info:
    git:
      mode: full
  endpoints:
    web:
      cors:
        allowed-origins: '*'
        allowed-methods: GET
      health:
        # https://stackoverflow.com/questions/46326527/spring-boot-actuator-endpoint-sensitivity-vs-security-enablement
        info:
          sensitive: false
      exposure:
        include:
          - info
          - beans
          - loggers
          - health

# https://springdoc.org/
# https://stackoverflow.com/questions/65820916/i-have-installed-openapi-3-using-springdoc-but-the-url-is-strange-can-i-change
springdoc:
  api-docs:
    path: /api-docs
  swagger-ui:
    path: /api-docs/ui

microservice-endpoints:
 main: http://deployapp-backend-service:8080

# for async processing with jobrunr
org:
  jobrunr:
    job-scheduler:
      enabled: true
    database:
      type: redis-lettuce
    background-job-server:
      worker_count: 2
      enabled: true
      poll-interval-in-seconds: 5
    dashboard:
      enabled: false

custom_jobrunr:
  keepalive-interval: PT30S

logging:
  level:
    org:
      springframework:
        web: DEBUG
        security: DEBUG
#          filter:
#            CommonsRequestLoggingFilter: DEBUG

server:
  tomcat:
    threads:
      min-spare: 5
      max: 20
  forward-headers-strategy: framework # https://github.com/springdoc/springdoc-openapi/issues/171
  servlet:
    session:
      cookie:
        # https://stackoverflow.com/questions/33095345/how-to-change-spring-session-redis-cookie-name
        name: "DEPLOYAPP_SESSION"
        path: /
  error:
    include-message: always
    include-binding-errors: always
    include-stacktrace: on_param
    include-exception: true


kubectl create configmap deployapp-storage-service --from-file=./application-prod.yml
Deployment YAML 
kind: Deployment
apiVersion: apps/v1
metadata:
  name: deployapp-storage-service
spec:
  replicas: 1
  selector:
    matchLabels:
      app: deployapp-storage-service
  template:
    metadata:
      labels:
        app: deployapp-storage-service
    spec:
      containers:
      - name: deployapp-storage-service
        image: transfusion/deployapp-storage-service:latest
        ports:
        - containerPort: 8080
        imagePullPolicy: Always
        env:
        - name: SPRING_PROFILES_ACTIVE
          value: "prod"
        - name: SPRING_CONFIG_LOCATION
          value: "file:///config/"
        resources: {}
        livenessProbe:
          httpGet:
            path: /actuator/health/liveness
            port: 8080
          initialDelaySeconds: 120 # because truffle environment takes forever to initialize
        readinessProbe:
          httpGet:
            path: /actuator/health/readiness
            port: 8080
          initialDelaySeconds: 120
        lifecycle:
          preStop:
            exec:
              command: ["sh", "-c", "sleep 10"]
        volumeMounts:
            - name: config-volume
              mountPath: /config
      volumes:
        - name: config-volume
          configMap:
            name: deployapp-storage-service
status: {}
kubectl apply -f deployapp-storage-service.yaml
K8s service 
kind: Service
apiVersion: v1
metadata:
  name: deployapp-storage-service
spec:
  type: NodePort
  ports:
    - port: 8080
      targetPort: 8080
      protocol: TCP
  selector:
    app: deployapp-storage-service
kubectl apply -f deployapp-storage-service.yaml

Frontend

Deployment YAML 
kind: Deployment
apiVersion: apps/v1
metadata:
  name: deployapp-frontend
spec:
  replicas: 1
  selector:
    matchLabels:
      app: deployapp-frontend
  template:
    metadata:
      labels:
        app: deployapp-frontend
    spec:
      containers:
      - name: deployapp-frontend
        image: transfusion/deployapp-frontend:latest-prod
        ports:
        - containerPort: 80
        imagePullPolicy: Always
kubectl apply -f deployapp-frontend.yaml
K8s Service 
kind: Service
apiVersion: v1
metadata:
  name: deployapp-frontend-service
spec:
  type: NodePort
  ports:
    - port: 80
      targetPort: 80
      protocol: TCP
  selector:
    app: deployapp-frontend
kubectl apply -f deployapp-frontend-service.yaml

Ingress

Ingress resource 
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt-production
    kubernetes.io/ingress.class: nginx
    nginx.ingress.kubernetes.io/proxy-body-size: 2g
    nginx.ingress.kubernetes.io/proxy-connect-timeout: 1h
    nginx.ingress.kubernetes.io/proxy-read-timeout: 1h
    nginx.ingress.kubernetes.io/proxy-send-timeout: 1h
    nginx.ingress.kubernetes.io/rewrite-target: /$2
    nginx.ingress.kubernetes.io/server-snippet: keepalive_timeout 1h; client_body_timeout
      1h;
    nginx.ingress.kubernetes.io/x-forwarded-prefix: /$1
  name: example-ingress
  namespace: default
spec:
  rules:
  - host: deploy.plan.ovh
    http:
      paths:
      - backend:
          service:
            name: deployapp-frontend-service
            port:
              number: 80
        path: /()(.*)
        pathType: Prefix
  - host: api.deploy.plan.ovh
    http:
      paths:
      - backend:
          service:
            name: deployapp-backend-service
            port:
              number: 8080
        path: /()(.*)
        pathType: Prefix
      - backend:
          service:
            name: deployapp-storage-service
            port:
              number: 8080
        path: /(storage)/(.*)
        pathType: Prefix
  tls:
  - hosts:
    - deploy.plan.ovh
    - api.deploy.plan.ovh
    secretName: letsencrypt-production

kubectl apply -f example-ingress.yaml
Clone this wiki locally