From a7dad7902c7b2db6648033fcdfe508336412c9b5 Mon Sep 17 00:00:00 2001 From: Thanayut Seethongchuen Date: Wed, 28 Aug 2024 18:05:52 +0200 Subject: [PATCH 1/6] enabled, canary, logPipeline, and ciServiceAccount This change make it possible to deploy all crab related services in single helm chart. --- .gitignore | 4 +- helm/crabserver/Chart.yaml | 4 +- helm/crabserver/apply_logstash.sh | 6 + helm/crabserver/config/preprod/config.py | 49 ++++ .../config/preprod/logstash}/logstash.conf | 4 +- .../config/preprod/logstash}/logstash.yml | 0 helm/crabserver/config/prod/config.py | 47 +++ .../config/prod/logstash/logstash.conf | 272 ++++++++++++++++++ .../config/prod/logstash/logstash.yml | 7 + helm/crabserver/config/test/config.py | 51 ++++ .../config/test/logstash/logstash.conf | 272 ++++++++++++++++++ .../config/test/logstash/logstash.yml | 7 + helm/crabserver/scripts/setup.sh | 34 +++ .../templates/ci-serviceaccount-rbac.yaml | 25 ++ .../templates/ci-serviceaccount-secret.yaml | 11 + .../templates/ci-serviceaccount.yaml | 8 + helm/crabserver/templates/configMap.yaml | 59 +--- .../templates/deployment-canary.yaml | 119 ++++++++ helm/crabserver/templates/deployment.yaml | 8 +- .../templates/filebeat-configmap.yaml | 37 +++ ...deamonset.yaml => filebeat-daemonset.yaml} | 15 +- helm/crabserver/templates/hpa.yaml | 2 +- helm/crabserver/templates/ingress.yaml | 2 +- .../templates/logstash-configmap.yaml | 29 ++ .../templates/logstash-deployment.yaml | 61 ++-- .../templates/logstash-service.yaml | 17 ++ helm/crabserver/templates/service.yaml | 3 +- helm/crabserver/templates/serviceaccount.yaml | 2 +- helm/crabserver/values-canary.yaml | 19 ++ helm/crabserver/values-genall.yaml | 17 ++ helm/crabserver/values-logpipeline.yaml | 6 + helm/crabserver/values-preprod-pypi.yaml | 18 -- helm/crabserver/values-prod-pypi.yaml | 18 -- helm/crabserver/values-prod.yaml | 3 +- helm/crabserver/values-test11-pypi.yaml | 23 -- helm/crabserver/values-test12.yaml | 2 +- .../{values-test1.yaml => values-test14.yaml} | 0 helm/crabserver/values-testbed-pypi.yaml | 18 -- helm/crabserver/values-testbed.yaml | 2 +- helm/crabserver/values.yaml | 104 ++++--- kubernetes/cmsweb/monitoring/crab/README.md | 53 ---- 41 files changed, 1171 insertions(+), 267 deletions(-) create mode 100644 helm/crabserver/apply_logstash.sh create mode 100644 helm/crabserver/config/preprod/config.py rename {kubernetes/cmsweb/monitoring/crab => helm/crabserver/config/preprod/logstash}/logstash.conf (99%) rename {kubernetes/cmsweb/monitoring/crab => helm/crabserver/config/preprod/logstash}/logstash.yml (100%) create mode 100644 helm/crabserver/config/prod/config.py create mode 100644 helm/crabserver/config/prod/logstash/logstash.conf create mode 100644 helm/crabserver/config/prod/logstash/logstash.yml create mode 100644 helm/crabserver/config/test/config.py create mode 100644 helm/crabserver/config/test/logstash/logstash.conf create mode 100644 helm/crabserver/config/test/logstash/logstash.yml create mode 100644 helm/crabserver/scripts/setup.sh create mode 100644 helm/crabserver/templates/ci-serviceaccount-rbac.yaml create mode 100644 helm/crabserver/templates/ci-serviceaccount-secret.yaml create mode 100644 helm/crabserver/templates/ci-serviceaccount.yaml create mode 100644 helm/crabserver/templates/deployment-canary.yaml create mode 100644 helm/crabserver/templates/filebeat-configmap.yaml rename helm/crabserver/templates/{deployment-deamonset.yaml => filebeat-daemonset.yaml} (74%) create mode 100644 helm/crabserver/templates/logstash-configmap.yaml rename kubernetes/cmsweb/monitoring/crab/logstash.yaml => helm/crabserver/templates/logstash-deployment.yaml (56%) create mode 100644 helm/crabserver/templates/logstash-service.yaml create mode 100644 helm/crabserver/values-canary.yaml create mode 100644 helm/crabserver/values-genall.yaml create mode 100644 helm/crabserver/values-logpipeline.yaml delete mode 100644 helm/crabserver/values-preprod-pypi.yaml delete mode 100644 helm/crabserver/values-prod-pypi.yaml delete mode 100644 helm/crabserver/values-test11-pypi.yaml rename helm/crabserver/{values-test1.yaml => values-test14.yaml} (100%) delete mode 100644 helm/crabserver/values-testbed-pypi.yaml delete mode 100644 kubernetes/cmsweb/monitoring/crab/README.md diff --git a/.gitignore b/.gitignore index e37b30a13..d72e57b25 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,7 @@ hmac # 3rd party packages kubernetes-prometheus -# docker area +# docker area docker/sqoop/awg-ETL-crons/ # skip configs/secrets @@ -81,3 +81,5 @@ kubernetes/rucio/futureprod-db.yaml kubernetes/rucio/dev_grafana_password.txt + +!helm/crabserver/config diff --git a/helm/crabserver/Chart.yaml b/helm/crabserver/Chart.yaml index 507ceccdc..c608764b3 100644 --- a/helm/crabserver/Chart.yaml +++ b/helm/crabserver/Chart.yaml @@ -15,9 +15,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.11 +version: 0.2.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. -appVersion: "v3.240731" +appVersion: "v3.240809" diff --git a/helm/crabserver/apply_logstash.sh b/helm/crabserver/apply_logstash.sh new file mode 100644 index 000000000..aa6ec8b22 --- /dev/null +++ b/helm/crabserver/apply_logstash.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -x +set -euo pipefail +ENV=test +kubectl create configmap logstash-crab --from-file=config/${ENV}/logstash/logstash.conf --from-file config/${ENV}/logstash/logstash.yml --dry-run=client -oyaml | kubectl apply -f - +kubectl delete pod $(kubectl get pod --no-headers -o custom-columns=":metadata.name" | grep logstash) diff --git a/helm/crabserver/config/preprod/config.py b/helm/crabserver/config/preprod/config.py new file mode 100644 index 000000000..6d717a0ae --- /dev/null +++ b/helm/crabserver/config/preprod/config.py @@ -0,0 +1,49 @@ +from WMCore.Configuration import Configuration +import re +import socket +import time +import os + +myhost = socket.getfqdn().lower() + +conf = Configuration() +main = conf.section_('main') +srv = main.section_('server') +srv.thread_pool = int(os.getenv('CRABSERVER_THREAD_POOL', 15)) +main.application = 'crabserver' +main.port = 8270 +main.index = 'data' + +main.authz_defaults = { 'role': None, 'group': None, 'site': None } +main.section_('tools').section_('cms_auth').key_file = "%s/auth/crabserver/header-auth-key" % __file__.rsplit('/', 3)[0] + +app = conf.section_('crabserver') +app.admin = 'cms-service-webtools@cern.ch' +app.description = 'CRABServer RESTFull API' +app.title = 'CRABRESTFull' + +views = conf.section_('views') +ui = views.section_('ui') +ui.object = 'CRABInterface.Pages.FrontPage.FrontPage' + +data = views.section_('data') +data.object = 'CRABInterface.RESTBaseAPI.RESTBaseAPI' +data.phedexurl = 'https://cmsweb.cern.ch/phedex/datasvc/xml/prod/' +data.dbsurl = 'http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet' +data.defaultBlacklist = ['T0_CH_CERN'] +data.serverhostcert = "%s/auth/crabserver/dmwm-service-cert.pem" % __file__.rsplit('/', 3)[0] +data.serverhostkey = "%s/auth/crabserver/dmwm-service-key.pem" % __file__.rsplit('/', 3)[0] +data.credpath = '%s/state/crabserver/proxy/' % __file__.rsplit('/', 4)[0] +data.backend = 'oracle' +data.db = 'CRABServerAuth.dbconfig' +data.s3 = 'CRABServerAuth.s3' +data.workflowManager = 'HTCondorDataWorkflow' + +data.extconfigurl = 'http://gitlab.cern.ch/crab3/CRAB3ServerConfig/raw/master/cmsweb-rest-config.json' + +data.loggingLevel = 10 +data.loggingFile = '%s/logs/crabserver/CRAB-%s.log' % (__file__.rsplit('/', 4)[0], myhost) +data.keptLogDays = 7 +data.mode = "cmsweb-preprod" + +data.delegateDN = "/DC=ch/DC=cern/OU=computers/CN=crab-(preprod|prod)-tw(01|02).cern.ch|/DC=ch/DC=cern/OU=computers/CN=crab-dev-tw(01|02|03|04).cern.ch|/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cmscrab/CN=(817881|373708)/CN=Robot: cms crab|/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=crabint1/CN=373708/CN=Robot: CMS CRAB Integration 1" diff --git a/kubernetes/cmsweb/monitoring/crab/logstash.conf b/helm/crabserver/config/preprod/logstash/logstash.conf similarity index 99% rename from kubernetes/cmsweb/monitoring/crab/logstash.conf rename to helm/crabserver/config/preprod/logstash/logstash.conf index b191f9ccd..e47fb7852 100644 --- a/kubernetes/cmsweb/monitoring/crab/logstash.conf +++ b/helm/crabserver/config/preprod/logstash/logstash.conf @@ -176,8 +176,8 @@ filter { } grok { - # [07/Dec/2022:20:06:47] crabserver-56c69685b6-477jr 188.184.91.103:41924 "GET /crabserver/devthree/info HTTP/1.1" 200 OK [data: 9354 in 103 out 4870 us ] [auth: ok "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=tseethon/CN=856006/CN=Thanayut Seethongchuen" "" ] [ref: "https://cmsweb-test12.cern.ch" "Mozilla/5.0 (X11; Linux x86_64; rv:107.0) Gecko/20100101 Firefox/107.0" ] Type=cherrypylog -+ match => { "message" => '\[%{NOTSPACE:timestamp_temp}\] %{DATA:backend} %{IPORHOST:clientip}(|:%{NUMBER}) "%{WORD:method} %{NOTSPACE:request} %{DATA:httpversion}" %{NUMBER:code:int} %{DATA} \[data: (-|%{NUMBER:bytes_sent:int}) in (-|%{NUMBER:bytes_received:int}) out (-|%{NUMBER:time_spent_ms:int}) us \] \[auth: %{DATA} "%{DATA:dn}".*\] \[ref: "%{DATA}.*" "%{DATA:client}" \]' } + # [07/Dec/2022:20:06:47] crabserver-56c69685b6-477jr 188.184.91.103:41924 "GET /crabserver/devthree/info HTTP/1.1" 200 OK [data: 9354 in 103 out 4870 us ] [auth: ok "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=tseethon/CN=856006/CN=Thanayut Seethongchuen" "" ] [ref: "https://cmsweb-test12.cern.ch" "Mozilla/5.0 (X11; Linux x86_64; rv:107.0) Gecko/20100101 Firefox/107.0" ] - Podname=crabserver-6cf49bcfff-bxvb6 Type=cherrypylog + match => { "message" => '\[%{NOTSPACE:timestamp_temp}\] %{DATA:backend} %{IPORHOST:clientip}(|:%{NUMBER}) "%{WORD:method} %{NOTSPACE:request} %{DATA:httpversion}" %{NUMBER:code:int} %{DATA} \[data: (-|%{NUMBER:bytes_sent:int}) in (-|%{NUMBER:bytes_received:int}) out (-|%{NUMBER:time_spent_ms:int}) us \] \[auth: %{DATA} "%{DATA:dn}".*\] \[ref: "%{DATA}.*" "%{DATA:client}" \]' } } grok { match => { "request" => '/%{WORD:system}%{UNIXPATH:uri_path}%{URIPARAM:uri_params}?' } diff --git a/kubernetes/cmsweb/monitoring/crab/logstash.yml b/helm/crabserver/config/preprod/logstash/logstash.yml similarity index 100% rename from kubernetes/cmsweb/monitoring/crab/logstash.yml rename to helm/crabserver/config/preprod/logstash/logstash.yml diff --git a/helm/crabserver/config/prod/config.py b/helm/crabserver/config/prod/config.py new file mode 100644 index 000000000..215f66ce0 --- /dev/null +++ b/helm/crabserver/config/prod/config.py @@ -0,0 +1,47 @@ +from WMCore.Configuration import Configuration +import re +import socket +import time +import os + +myhost = socket.getfqdn().lower() + +conf = Configuration() +main = conf.section_('main') +srv = main.section_('server') +srv.thread_pool = int(os.getenv('CRABSERVER_THREAD_POOL', 15)) +main.application = 'crabserver' +main.port = 8270 +main.index = 'data' + +main.authz_defaults = { 'role': None, 'group': None, 'site': None } +main.section_('tools').section_('cms_auth').key_file = "%s/auth/crabserver/header-auth-key" % __file__.rsplit('/', 3)[0] + +app = conf.section_('crabserver') +app.admin = 'cms-service-webtools@cern.ch' +app.description = 'CRABServer RESTFull API' +app.title = 'CRABRESTFull' + +views = conf.section_('views') +ui = views.section_('ui') +ui.object = 'CRABInterface.Pages.FrontPage.FrontPage' + +data = views.section_('data') +data.object = 'CRABInterface.RESTBaseAPI.RESTBaseAPI' +data.phedexurl = 'https://cmsweb.cern.ch/phedex/datasvc/xml/prod/' +data.dbsurl = 'http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet' +data.defaultBlacklist = ['T0_CH_CERN'] +data.serverhostcert = "%s/auth/crabserver/dmwm-service-cert.pem" % __file__.rsplit('/', 3)[0] +data.serverhostkey = "%s/auth/crabserver/dmwm-service-key.pem" % __file__.rsplit('/', 3)[0] +data.credpath = '%s/state/crabserver/proxy/' % __file__.rsplit('/', 4)[0] +data.backend = 'oracle' +data.db = 'CRABServerAuth.dbconfig' +data.s3 = 'CRABServerAuth.s3' +data.workflowManager = 'HTCondorDataWorkflow' + +data.extconfigurl = 'http://gitlab.cern.ch/crab3/CRAB3ServerConfig/raw/master/cmsweb-rest-config.json' + +data.loggingLevel = 10 +data.loggingFile = '%s/logs/crabserver/CRAB-%s.log' % (__file__.rsplit('/', 4)[0], myhost) +data.keptLogDays = 7 +data.mode = "cmsweb-prod" diff --git a/helm/crabserver/config/prod/logstash/logstash.conf b/helm/crabserver/config/prod/logstash/logstash.conf new file mode 100644 index 000000000..e47fb7852 --- /dev/null +++ b/helm/crabserver/config/prod/logstash/logstash.conf @@ -0,0 +1,272 @@ +# Migrated from: https://github.com/dmwm/CRABServer/blob/master/src/script/Monitor/logstash/crabtaskworker.conf + +input { beats { port => 5044 } } + +filter { + # whitelisted tags + if "crabtaskworker" not in [tags] and "crabpublisher" not in [tags] and "crabhttpcall" not in [tags] and "crabrest" not in [tags] { drop { } } + + mutate { + # ---- Common fields ---- [metadata] + add_field => { + "cmsweb_cluster" => "${CMSWEB_CLUSTER:NA}" + "cmsweb_env" => "${CMSWEB_ENV:NA}" + "filebeat_id" => "%{[agent][id]}" + "filebeat_name" => "%{[agent][name]}" + "filebeat_version" => "%{[agent][version]}" + "hostname" => "%{[host][name]}" + "log_file" => "%{[log][file][path]}" + } + } + + if "crabtaskworker" in [tags] { + # ------------------------------------------------------------------------------------------------------------------------------------------------------------------ + # TASKWORKER + mutate { + # MONIT mandatory fields. Be aware that you cannot use any of these names in grok/mutate parsers! + replace => { + "producer" => "crab" + "type" => "crabtaskworker" + } + } + if "completed in" in [message] { + grok { + match => { + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:DEBUG:Worker,%{INT}:Process-%{NOTSPACE:slaveID}: %{NOTSPACE:workType} work on %{NOTSPACE:taskName} completed in %{NUMBER:completionTime:int} seconds" + } + add_field => {"log_type" => "work_on_task_completed"} + overwrite => ["message"] + } + + } else if ( [message] =~ /.*Starting.*at.*on.*/ ) { + grok { + match => { + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:DEBUG:Worker,%{INT}:Process-%{NOTSPACE:slaveID}: Starting on %{NOTSPACE:taskName}" + } + add_field => {"log_type" => "start_new_task"} + overwrite => ["message"] + } + } else if "TWSTART" in [message] { + grok { + match => { + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:MasterWorker,%{INT}:TWSTART: %{GREEDYDATA:tw_json_data}" + } + add_field => {"log_type" => "tw_config_data"} + overwrite => ["message"] + } + if [tw_json_data] { + json { + source => "tw_json_data" + target => "tw_json_data" + } + } + + } else if ( [message] =~ /.*Finished.*object at .*/ ) { + grok { + match => { + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:INFO:Handler,%{INT}:Finished on %{NOTSPACE:taskName} in %{NUMBER:completionTime:int} seconds" + } + add_field => {"log_type" => "action_on_task_finished"} + overwrite => ["message"] + } + } else { drop {} } + + date { + # Since date string does not provide timezone, we need to specify it explicitly + match => ["timestamp_temp", "YYYY-MM-dd HH:mm:ss,SSS"] + timezone => "Europe/Zurich" + target => "rec_timestamp_str" + } + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + } else if "crabpublisher" in [tags] { + # ------------------------------------------------------------------------------------------------------------------------------------------------------------------ + # CRABPUBLISHER + mutate { + # MONIT mandatory fields. Be aware that you cannot use any of these names in grok/mutate parsers! + replace => { + "producer" => "crab" + "type" => "crabpublisher" + } + } + + if "PUBSTART" in [message] { + grok { + match => { + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:PUBSTART:%{GREEDYDATA:pb_json_data}" + } + add_field => {"log_type" => "publisher_config_data"} + overwrite => ["message"] + } + if [pb_json_data] { + json{ + source => "pb_json_data" + target => "pb_json_data" + } + } + + } else if ( [message] =~ /.*blocks failed.*files.*/ ) { + grok { + match => { + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:Taskname %{NOTSPACE:taskName} : %{INT:blocks} blocks failed for a total of %{INT:files} files" + } + add_field => {"log_type" => "failed_publication"} + overwrite => ["message"] + } + + } else if ( [message] =~ /.*Published.*files in.*/ ) { + grok { + match => { + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:Taskname %{NOTSPACE:taskName} is %{NOTSPACE:publicationResult}. Published %{INT:filesPublished} files in %{INT:blocks} blocks." + } + add_field => {"log_type" => "successful_publication"} + overwrite => ["message"] + } + + } else if ( [message] =~ /.*Exception.*TaskPublish.*/ ) { + grok { + match => { + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:%{GREEDYDATA:exceptionHandled}" + } + add_field => {"log_type" => "publication_error"} + overwrite => ["message"] + } + + # This filter is currently used for production publisher. After we deploy preprod to prod, this filter can be dropped + } else if "DEBUG:master" in [message] { + grok { + match => { + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:master:%{SPACE}%{INT:acquiredFiles}%{SPACE}:%{SPACE}%{NOTSPACE:taskName}" + } + add_field => {"log_type" => "acquired_files"} + overwrite => ["message"] + } + + # this filter mathces changes in #6861, which is already in preprod and dev + } else if "acquired_files" in [message] { + grok { + match => { + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:acquired_files:%{SPACE}%{NOTSPACE:acquiredFilesStatus}%{SPACE}%{INT:acquiredFiles}%{SPACE}:%{SPACE}%{NOTSPACE:taskName}" + } + add_field => {"log_type" => "acquired_files_status"} + overwrite => ["message"] + } + + } else { drop { } } + date { + # Since date string does not provide timezone, we need to specify it explicitly + match => ["timestamp_temp", "YYYY-MM-dd HH:mm:ss,SSS"] + timezone => "Europe/Zurich" + target => "rec_timestamp_str" + } + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + } else if "crabhttpcall" in [tags] { + # ------------------------------------------------------------------------------------------------------------------------------------------------------------------ + # CRABHTTPCALL (previously crabserver) + mutate { + # MONIT mandatory fields. Be aware that you cannot use any of these names in grok/mutate parsers! + replace => { + "producer" => "crab" + "type" => "crabhttpcall" + } + } + # parse log path to get pod name + # /var/log/pods/crab_crabserver-test-5f9bd5f58d-5nfhc_5366ebdb-ee96-4641-8643-f3bbe63f808f/crabserver/0.log + grok { + match => { "[log][file][path]" => '/var/log/pods/crab_%{DATA:pod_name}_%{DATA}/%{GREEDYDATA}' } + } + + grok { + # [07/Dec/2022:20:06:47] crabserver-56c69685b6-477jr 188.184.91.103:41924 "GET /crabserver/devthree/info HTTP/1.1" 200 OK [data: 9354 in 103 out 4870 us ] [auth: ok "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=tseethon/CN=856006/CN=Thanayut Seethongchuen" "" ] [ref: "https://cmsweb-test12.cern.ch" "Mozilla/5.0 (X11; Linux x86_64; rv:107.0) Gecko/20100101 Firefox/107.0" ] - Podname=crabserver-6cf49bcfff-bxvb6 Type=cherrypylog + match => { "message" => '\[%{NOTSPACE:timestamp_temp}\] %{DATA:backend} %{IPORHOST:clientip}(|:%{NUMBER}) "%{WORD:method} %{NOTSPACE:request} %{DATA:httpversion}" %{NUMBER:code:int} %{DATA} \[data: (-|%{NUMBER:bytes_sent:int}) in (-|%{NUMBER:bytes_received:int}) out (-|%{NUMBER:time_spent_ms:int}) us \] \[auth: %{DATA} "%{DATA:dn}".*\] \[ref: "%{DATA}.*" "%{DATA:client}" \]' } + } + grok { + match => { "request" => '/%{WORD:system}%{UNIXPATH:uri_path}%{URIPARAM:uri_params}?' } + } + if ![uri_params] { + ruby { code => "event.set('uri_params','')" } + } + grok { match => { "uri_path" => '/.*/%{DATA:api}$' } } + if [client] { + grok { match => { "client" => '%{DATA:client_name}/%{DATA:client_version}$' } } + } + # !ATTENTION! Different date-time format than publisher and TW, means that special for crabserver. + date { + # Since date string does not provide timezone, we need to specify it explicitly + match => [ "timestamp_temp", "dd/MMM/yyyy:HH:mm:ss" ] + timezone => "Europe/Zurich" + target => "rec_timestamp_str" + } + mutate { gsub => [ "dn","/CN=\d+","" ] } + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + } else if "crabrest" in [tags] { + # ------------------------------------------------------------------------------------------------------------------------------------------------------------------ + # CRABLOG + mutate { + # MONIT mandatory fields. Be aware that you cannot use any of these names in grok/mutate parsers! + replace => { + "producer" => "crab" + "type" => "crabrest" + } + } + # parse log path to get pod name + # /var/log/pods/crab_crabserver-test-5f9bd5f58d-5nfhc_5366ebdb-ee96-4641-8643-f3bbe63f808f/crabserver/0.log + grok { + match => { "[log][file][path]" => '/var/log/pods/crab_%{DATA:pod_name}_%{DATA}/%{GREEDYDATA}' } + } + + if ( [message] =~ /.*MeasureTime:seconds.*/ ) { + grok { + match => { "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:trace_id}:%{NOTSPACE:log_level}:%{NOTSPACE}:MeasureTime:seconds - modulename=%{NOTSPACE:modulename} label='%{NOTSPACE:label}' tot=%{NUMBER:tot:float} proc=%{NUMBER:proc:float} thread=%{NUMBER:thread:float}" } + add_field => {"log_type" => "measure_time"} + } + } else if ( [message] =~ /.*MeasureSize:bytes.*/ ) { + grok { + match => { "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:trace_id}:%{NOTSPACE:log_level}:%{NOTSPACE}:MeasureSize:bytes - modulename=%{NOTSPACE:modulename} label='%{NOTSPACE:label}' obj_size=%{INT:obj_size:int} get_size_time=%{NUMBER:get_size_time:float}" } + add_field => {"log_type" => "measure_size"} + } + } + date { + # Since date string does not provide timezone, we need to specify it explicitly + match => ["timestamp_temp", "YYYY-MM-dd HH:mm:ss,SSS"] + timezone => "Europe/Zurich" + target => "rec_timestamp_str" + } + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + } + + # ---- Common modifications ---- + if "_grokparsefailure" in [tags] { drop { } } + + # Date operations + ruby { + # "metadata.timestamp"(date in ES) equals to record_time(int in ES) which we need for latency calculation + # "producer_time" is filebeat process time stamp. + code => "event.set('timestamp', (event.get('rec_timestamp_str').to_f * 1000).to_i) + event.set('record_time', (event.get('rec_timestamp_str').to_f * 1000).to_i) + event.set('producer_time', (event.get('@timestamp').to_f * 1000).to_i) + " + } + + # remove quotes from message entry since it will break the JSON + mutate { gsub => [ "message", "\n", "", "message", "\"", ""] } + + # remove undesired fields and objects + mutate { remove_field => ["@timestamp", "@version", "timestamp_temp", "agent", "log", "input", "tags", "ecs", "host", "cloud", "event" ] } + +} + +output { + if [type] in ["crabtaskworker", "crabpublisher", "crabhttpcall", "crabrest"] { + http { + http_method => "post" + url => "http://monit-logs.cern.ch:10012/" + content_type => "application/json; charset=UTF-8" + format => "json_batch" + socket_timeout => 60 + connect_timeout => 60 + } + } else { + # For debugging, please keep it to catch problems + stdout { codec => json } + } +} diff --git a/helm/crabserver/config/prod/logstash/logstash.yml b/helm/crabserver/config/prod/logstash/logstash.yml new file mode 100644 index 000000000..c826af0ea --- /dev/null +++ b/helm/crabserver/config/prod/logstash/logstash.yml @@ -0,0 +1,7 @@ +http.host: "0.0.0.0" +path.config: /usr/share/logstash/pipeline +# disable connection to ES +xpack.monitoring.enabled: false +pipeline.workers: 16 +pipeline.batch.size: 256 +pipeline.batch.delay: 200 diff --git a/helm/crabserver/config/test/config.py b/helm/crabserver/config/test/config.py new file mode 100644 index 000000000..3c3efb04f --- /dev/null +++ b/helm/crabserver/config/test/config.py @@ -0,0 +1,51 @@ +from WMCore.Configuration import Configuration +import re +import socket +import time +import os + +myhost = socket.getfqdn().lower() + +conf = Configuration() +main = conf.section_('main') +srv = main.section_('server') +srv.thread_pool = int(os.getenv('CRABSERVER_THREAD_POOL', 25)) +main.application = 'crabserver' +main.port = 8270 +main.index = 'data' + +main.authz_defaults = { 'role': None, 'group': None, 'site': None } +main.section_('tools').section_('cms_auth').key_file = "%s/auth/crabserver/header-auth-key" % __file__.rsplit('/', 3)[0] + +app = conf.section_('crabserver') +app.admin = 'cms-service-webtools@cern.ch' +app.description = 'CRABServer RESTFull API' +app.title = 'CRABRESTFull' + +views = conf.section_('views') +ui = views.section_('ui') +ui.object = 'CRABInterface.Pages.FrontPage.FrontPage' + +data = views.section_('data') +data.object = 'CRABInterface.RESTBaseAPI.RESTBaseAPI' +data.phedexurl = 'https://cmsweb.cern.ch/phedex/datasvc/xml/prod/' +data.dbsurl = 'http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet' +data.defaultBlacklist = ['T0_CH_CERN'] +data.serverhostcert = "%s/auth/crabserver/dmwm-service-cert.pem" % __file__.rsplit('/', 3)[0] +data.serverhostkey = "%s/auth/crabserver/dmwm-service-key.pem" % __file__.rsplit('/', 3)[0] +data.credpath = '%s/state/crabserver/proxy/' % __file__.rsplit('/', 4)[0] +data.backend = 'oracle' +data.db = 'CRABServerAuth.dbconfig' +data.s3 = 'CRABServerAuth.s3' +data.workflowManager = 'HTCondorDataWorkflow' + +data.extconfigurl = 'http://gitlab.cern.ch/crab3/CRAB3ServerConfig/raw/master/cmsweb-rest-config.json' + +data.loggingLevel = 10 +data.loggingFile = '%s/logs/crabserver/CRAB-%s.log' % (__file__.rsplit('/', 4)[0], myhost) +data.keptLogDays = 7 +data.mode = "cmsweb-test" + +data.enableQueryLoadAllRows = os.getenv('CRABSERVER_ENABLE_QUERY_LOAD_ALL_ROWS', 'True').lower() in ('true', '1', 't') + +data.delegateDN = "/DC=ch/DC=cern/OU=computers/CN=crab-(preprod|prod)-tw(01|02).cern.ch|/DC=ch/DC=cern/OU=computers/CN=crab-dev-tw(01|02|03|04).cern.ch|/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cmscrab/CN=(817881|373708)/CN=Robot: cms crab|/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=crabint1/CN=373708/CN=Robot: CMS CRAB Integration 1" diff --git a/helm/crabserver/config/test/logstash/logstash.conf b/helm/crabserver/config/test/logstash/logstash.conf new file mode 100644 index 000000000..e47fb7852 --- /dev/null +++ b/helm/crabserver/config/test/logstash/logstash.conf @@ -0,0 +1,272 @@ +# Migrated from: https://github.com/dmwm/CRABServer/blob/master/src/script/Monitor/logstash/crabtaskworker.conf + +input { beats { port => 5044 } } + +filter { + # whitelisted tags + if "crabtaskworker" not in [tags] and "crabpublisher" not in [tags] and "crabhttpcall" not in [tags] and "crabrest" not in [tags] { drop { } } + + mutate { + # ---- Common fields ---- [metadata] + add_field => { + "cmsweb_cluster" => "${CMSWEB_CLUSTER:NA}" + "cmsweb_env" => "${CMSWEB_ENV:NA}" + "filebeat_id" => "%{[agent][id]}" + "filebeat_name" => "%{[agent][name]}" + "filebeat_version" => "%{[agent][version]}" + "hostname" => "%{[host][name]}" + "log_file" => "%{[log][file][path]}" + } + } + + if "crabtaskworker" in [tags] { + # ------------------------------------------------------------------------------------------------------------------------------------------------------------------ + # TASKWORKER + mutate { + # MONIT mandatory fields. Be aware that you cannot use any of these names in grok/mutate parsers! + replace => { + "producer" => "crab" + "type" => "crabtaskworker" + } + } + if "completed in" in [message] { + grok { + match => { + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:DEBUG:Worker,%{INT}:Process-%{NOTSPACE:slaveID}: %{NOTSPACE:workType} work on %{NOTSPACE:taskName} completed in %{NUMBER:completionTime:int} seconds" + } + add_field => {"log_type" => "work_on_task_completed"} + overwrite => ["message"] + } + + } else if ( [message] =~ /.*Starting.*at.*on.*/ ) { + grok { + match => { + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:DEBUG:Worker,%{INT}:Process-%{NOTSPACE:slaveID}: Starting on %{NOTSPACE:taskName}" + } + add_field => {"log_type" => "start_new_task"} + overwrite => ["message"] + } + } else if "TWSTART" in [message] { + grok { + match => { + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:MasterWorker,%{INT}:TWSTART: %{GREEDYDATA:tw_json_data}" + } + add_field => {"log_type" => "tw_config_data"} + overwrite => ["message"] + } + if [tw_json_data] { + json { + source => "tw_json_data" + target => "tw_json_data" + } + } + + } else if ( [message] =~ /.*Finished.*object at .*/ ) { + grok { + match => { + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:INFO:Handler,%{INT}:Finished on %{NOTSPACE:taskName} in %{NUMBER:completionTime:int} seconds" + } + add_field => {"log_type" => "action_on_task_finished"} + overwrite => ["message"] + } + } else { drop {} } + + date { + # Since date string does not provide timezone, we need to specify it explicitly + match => ["timestamp_temp", "YYYY-MM-dd HH:mm:ss,SSS"] + timezone => "Europe/Zurich" + target => "rec_timestamp_str" + } + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + } else if "crabpublisher" in [tags] { + # ------------------------------------------------------------------------------------------------------------------------------------------------------------------ + # CRABPUBLISHER + mutate { + # MONIT mandatory fields. Be aware that you cannot use any of these names in grok/mutate parsers! + replace => { + "producer" => "crab" + "type" => "crabpublisher" + } + } + + if "PUBSTART" in [message] { + grok { + match => { + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:PUBSTART:%{GREEDYDATA:pb_json_data}" + } + add_field => {"log_type" => "publisher_config_data"} + overwrite => ["message"] + } + if [pb_json_data] { + json{ + source => "pb_json_data" + target => "pb_json_data" + } + } + + } else if ( [message] =~ /.*blocks failed.*files.*/ ) { + grok { + match => { + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:Taskname %{NOTSPACE:taskName} : %{INT:blocks} blocks failed for a total of %{INT:files} files" + } + add_field => {"log_type" => "failed_publication"} + overwrite => ["message"] + } + + } else if ( [message] =~ /.*Published.*files in.*/ ) { + grok { + match => { + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:Taskname %{NOTSPACE:taskName} is %{NOTSPACE:publicationResult}. Published %{INT:filesPublished} files in %{INT:blocks} blocks." + } + add_field => {"log_type" => "successful_publication"} + overwrite => ["message"] + } + + } else if ( [message] =~ /.*Exception.*TaskPublish.*/ ) { + grok { + match => { + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:%{GREEDYDATA:exceptionHandled}" + } + add_field => {"log_type" => "publication_error"} + overwrite => ["message"] + } + + # This filter is currently used for production publisher. After we deploy preprod to prod, this filter can be dropped + } else if "DEBUG:master" in [message] { + grok { + match => { + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:master:%{SPACE}%{INT:acquiredFiles}%{SPACE}:%{SPACE}%{NOTSPACE:taskName}" + } + add_field => {"log_type" => "acquired_files"} + overwrite => ["message"] + } + + # this filter mathces changes in #6861, which is already in preprod and dev + } else if "acquired_files" in [message] { + grok { + match => { + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:acquired_files:%{SPACE}%{NOTSPACE:acquiredFilesStatus}%{SPACE}%{INT:acquiredFiles}%{SPACE}:%{SPACE}%{NOTSPACE:taskName}" + } + add_field => {"log_type" => "acquired_files_status"} + overwrite => ["message"] + } + + } else { drop { } } + date { + # Since date string does not provide timezone, we need to specify it explicitly + match => ["timestamp_temp", "YYYY-MM-dd HH:mm:ss,SSS"] + timezone => "Europe/Zurich" + target => "rec_timestamp_str" + } + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + } else if "crabhttpcall" in [tags] { + # ------------------------------------------------------------------------------------------------------------------------------------------------------------------ + # CRABHTTPCALL (previously crabserver) + mutate { + # MONIT mandatory fields. Be aware that you cannot use any of these names in grok/mutate parsers! + replace => { + "producer" => "crab" + "type" => "crabhttpcall" + } + } + # parse log path to get pod name + # /var/log/pods/crab_crabserver-test-5f9bd5f58d-5nfhc_5366ebdb-ee96-4641-8643-f3bbe63f808f/crabserver/0.log + grok { + match => { "[log][file][path]" => '/var/log/pods/crab_%{DATA:pod_name}_%{DATA}/%{GREEDYDATA}' } + } + + grok { + # [07/Dec/2022:20:06:47] crabserver-56c69685b6-477jr 188.184.91.103:41924 "GET /crabserver/devthree/info HTTP/1.1" 200 OK [data: 9354 in 103 out 4870 us ] [auth: ok "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=tseethon/CN=856006/CN=Thanayut Seethongchuen" "" ] [ref: "https://cmsweb-test12.cern.ch" "Mozilla/5.0 (X11; Linux x86_64; rv:107.0) Gecko/20100101 Firefox/107.0" ] - Podname=crabserver-6cf49bcfff-bxvb6 Type=cherrypylog + match => { "message" => '\[%{NOTSPACE:timestamp_temp}\] %{DATA:backend} %{IPORHOST:clientip}(|:%{NUMBER}) "%{WORD:method} %{NOTSPACE:request} %{DATA:httpversion}" %{NUMBER:code:int} %{DATA} \[data: (-|%{NUMBER:bytes_sent:int}) in (-|%{NUMBER:bytes_received:int}) out (-|%{NUMBER:time_spent_ms:int}) us \] \[auth: %{DATA} "%{DATA:dn}".*\] \[ref: "%{DATA}.*" "%{DATA:client}" \]' } + } + grok { + match => { "request" => '/%{WORD:system}%{UNIXPATH:uri_path}%{URIPARAM:uri_params}?' } + } + if ![uri_params] { + ruby { code => "event.set('uri_params','')" } + } + grok { match => { "uri_path" => '/.*/%{DATA:api}$' } } + if [client] { + grok { match => { "client" => '%{DATA:client_name}/%{DATA:client_version}$' } } + } + # !ATTENTION! Different date-time format than publisher and TW, means that special for crabserver. + date { + # Since date string does not provide timezone, we need to specify it explicitly + match => [ "timestamp_temp", "dd/MMM/yyyy:HH:mm:ss" ] + timezone => "Europe/Zurich" + target => "rec_timestamp_str" + } + mutate { gsub => [ "dn","/CN=\d+","" ] } + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + } else if "crabrest" in [tags] { + # ------------------------------------------------------------------------------------------------------------------------------------------------------------------ + # CRABLOG + mutate { + # MONIT mandatory fields. Be aware that you cannot use any of these names in grok/mutate parsers! + replace => { + "producer" => "crab" + "type" => "crabrest" + } + } + # parse log path to get pod name + # /var/log/pods/crab_crabserver-test-5f9bd5f58d-5nfhc_5366ebdb-ee96-4641-8643-f3bbe63f808f/crabserver/0.log + grok { + match => { "[log][file][path]" => '/var/log/pods/crab_%{DATA:pod_name}_%{DATA}/%{GREEDYDATA}' } + } + + if ( [message] =~ /.*MeasureTime:seconds.*/ ) { + grok { + match => { "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:trace_id}:%{NOTSPACE:log_level}:%{NOTSPACE}:MeasureTime:seconds - modulename=%{NOTSPACE:modulename} label='%{NOTSPACE:label}' tot=%{NUMBER:tot:float} proc=%{NUMBER:proc:float} thread=%{NUMBER:thread:float}" } + add_field => {"log_type" => "measure_time"} + } + } else if ( [message] =~ /.*MeasureSize:bytes.*/ ) { + grok { + match => { "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:trace_id}:%{NOTSPACE:log_level}:%{NOTSPACE}:MeasureSize:bytes - modulename=%{NOTSPACE:modulename} label='%{NOTSPACE:label}' obj_size=%{INT:obj_size:int} get_size_time=%{NUMBER:get_size_time:float}" } + add_field => {"log_type" => "measure_size"} + } + } + date { + # Since date string does not provide timezone, we need to specify it explicitly + match => ["timestamp_temp", "YYYY-MM-dd HH:mm:ss,SSS"] + timezone => "Europe/Zurich" + target => "rec_timestamp_str" + } + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + } + + # ---- Common modifications ---- + if "_grokparsefailure" in [tags] { drop { } } + + # Date operations + ruby { + # "metadata.timestamp"(date in ES) equals to record_time(int in ES) which we need for latency calculation + # "producer_time" is filebeat process time stamp. + code => "event.set('timestamp', (event.get('rec_timestamp_str').to_f * 1000).to_i) + event.set('record_time', (event.get('rec_timestamp_str').to_f * 1000).to_i) + event.set('producer_time', (event.get('@timestamp').to_f * 1000).to_i) + " + } + + # remove quotes from message entry since it will break the JSON + mutate { gsub => [ "message", "\n", "", "message", "\"", ""] } + + # remove undesired fields and objects + mutate { remove_field => ["@timestamp", "@version", "timestamp_temp", "agent", "log", "input", "tags", "ecs", "host", "cloud", "event" ] } + +} + +output { + if [type] in ["crabtaskworker", "crabpublisher", "crabhttpcall", "crabrest"] { + http { + http_method => "post" + url => "http://monit-logs.cern.ch:10012/" + content_type => "application/json; charset=UTF-8" + format => "json_batch" + socket_timeout => 60 + connect_timeout => 60 + } + } else { + # For debugging, please keep it to catch problems + stdout { codec => json } + } +} diff --git a/helm/crabserver/config/test/logstash/logstash.yml b/helm/crabserver/config/test/logstash/logstash.yml new file mode 100644 index 000000000..c826af0ea --- /dev/null +++ b/helm/crabserver/config/test/logstash/logstash.yml @@ -0,0 +1,7 @@ +http.host: "0.0.0.0" +path.config: /usr/share/logstash/pipeline +# disable connection to ES +xpack.monitoring.enabled: false +pipeline.workers: 16 +pipeline.batch.size: 256 +pipeline.batch.delay: 200 diff --git a/helm/crabserver/scripts/setup.sh b/helm/crabserver/scripts/setup.sh new file mode 100644 index 000000000..995bace70 --- /dev/null +++ b/helm/crabserver/scripts/setup.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -x +set -euo pipefail + +# copy robotcert +sudo cp /etc/robots/robotkey.pem /data/srv/current/auth/crabserver/dmwm-service-key.pem +sudo cp /etc/robots/robotcert.pem /data/srv/current/auth/crabserver/dmwm-service-cert.pem +sudo chown $USER:$USER /data/srv/current/auth/crabserver/dmwm-service-key.pem +sudo chown $USER:$USER /data/srv/current/auth/crabserver/dmwm-service-cert.pem + +# hmac key +sudo cp /etc/hmac/hmac /data/srv/current/auth/crabserver/header-auth-key +sudo chown $USER:$USER /data/srv/current/auth/crabserver/header-auth-key + +# config.py +sudo cp /opt/config/config.py /data/srv/current/config/crabserver/config.py +sudo chown $USER:$USER /data/srv/current/config/crabserver/config.py + +# CRABServerAuth.py +sudo cp /etc/secrets/CRABServerAuth.py /data/srv/current/auth/crabserver/CRABServerAuth.py +sudo chown $USER:$USER /data/srv/current/auth/crabserver/CRABServerAuth.py + +# test X509_USER_PROXY +# Wa: I do not understand why we use this file instead of robotcert +ls /etc/proxy/proxy +export X509_USER_PROXY=/etc/proxy/proxy + +# grid security +sudo cp /host/etc/grid-security/* /etc/grid-security +echo 'INFO Files in /etc/grid-security' +ls -lahZ /etc/grid-security + +exec /usr/bin/tini -- /data/run.sh diff --git a/helm/crabserver/templates/ci-serviceaccount-rbac.yaml b/helm/crabserver/templates/ci-serviceaccount-rbac.yaml new file mode 100644 index 000000000..013f8eeaa --- /dev/null +++ b/helm/crabserver/templates/ci-serviceaccount-rbac.yaml @@ -0,0 +1,25 @@ +{{- if .Values.ciServiceAccount.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: crab-ns-full-access + namespace: crab +subjects: +- kind: ServiceAccount + name: crab-gitlab-ci + namespace: crab +roleRef: + kind: Role + name: crab-ns-full-access + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: crab + name: crab-ns-full-access +rules: +- apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] +{{- end }} diff --git a/helm/crabserver/templates/ci-serviceaccount-secret.yaml b/helm/crabserver/templates/ci-serviceaccount-secret.yaml new file mode 100644 index 000000000..fa5adccf9 --- /dev/null +++ b/helm/crabserver/templates/ci-serviceaccount-secret.yaml @@ -0,0 +1,11 @@ +{{- if .Values.ciServiceAccount.enabled -}} +apiVersion: v1 +kind: Secret +metadata: + annotations: + kubernetes.io/service-account.name: crab-gitlab-ci + name: crab-gitlab-ci-secret + labels: + {{- include "crabserver.labels" . | nindent 4 }} +type: kubernetes.io/service-account-token +{{- end }} \ No newline at end of file diff --git a/helm/crabserver/templates/ci-serviceaccount.yaml b/helm/crabserver/templates/ci-serviceaccount.yaml new file mode 100644 index 000000000..0319bc93b --- /dev/null +++ b/helm/crabserver/templates/ci-serviceaccount.yaml @@ -0,0 +1,8 @@ +{{- if .Values.ciServiceAccount.enabled -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: crab-gitlab-ci + labels: + {{- include "crabserver.labels" . | nindent 4 }} +{{- end }} diff --git a/helm/crabserver/templates/configMap.yaml b/helm/crabserver/templates/configMap.yaml index 102f53ea3..cdbfc4a4c 100644 --- a/helm/crabserver/templates/configMap.yaml +++ b/helm/crabserver/templates/configMap.yaml @@ -1,3 +1,5 @@ +{{- $environment := .Values.environment | default dict }} +{{- if .Values.enabled -}} kind: ConfigMap apiVersion: v1 metadata: @@ -6,48 +8,15 @@ metadata: app: crabserver namespace: crab data: - setup-certs-and-run.sh: | - #!/bin/bash - sudo cp /host/etc/grid-security/* /etc/grid-security - echo 'INFO Files in /etc/grid-security' - ls -lahZ /etc/grid-security - #su -c "cd /data && /data/run.sh" --preserve-environment _crabserver - if command -v tini &> /dev/null; then - /data/entrypoint.sh /data/run.sh - else - cd /data && /data/run.sh - fi ---- -apiVersion: v1 -kind: ConfigMap -metadata: - labels: - k8s-app: filebeat - name: filebeat-crab-config - namespace: crab -data: - filebeat.yml: |- - filebeat.inputs: - - type: container - paths: - - /var/log/pods/crab_crabserver*/crabserver/*.log - include_lines: - - '\] crabserver-\w+-\w+ \d' - tags: - - "crabhttpcall" - - type: container - paths: - - /var/log/pods/crab_crabserver*/crabserver/*.log - include_lines: - - 'MeasureTime:seconds - ' - - 'MeasureSize:bytes - ' - tags: - - "crabrest" - output.logstash: - hosts: ["logstash:5044"] - compression_level: 3 - bulk_max_size: 4096 - queue.mem: - events: 65536 - logging.metrics.enabled: false - logging.level: info + setup.sh: | +{{ (.Files.Get "scripts/setup.sh" | indent 4) }} + config.py: | +{{- if (eq (toString $environment) "prod") }} +{{ (.Files.Get "config/prod/config.py" | indent 4) }} +{{- else if (eq (toString $environment) "preprod")}} +{{ (.Files.Get "config/preprod/config.py" | indent 4) }} +{{- else }} +{{ (.Files.Get "config/test/config.py" | indent 4) }} +{{- end }} + +{{- end }} diff --git a/helm/crabserver/templates/deployment-canary.yaml b/helm/crabserver/templates/deployment-canary.yaml new file mode 100644 index 000000000..df833be62 --- /dev/null +++ b/helm/crabserver/templates/deployment-canary.yaml @@ -0,0 +1,119 @@ +{{- $environment := .Values.environment | default dict }} +{{- if and (.Values.enabled) (.Values.canary.enabled) }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: crabserver-canary + labels: + app: crabserver-canary + namespace: crab +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.canary.replicaCount }} + {{- end }} + selector: + matchLabels: + app: crabserver + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + app: crabserver + env: k8s-{{.Values.environment}} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: crabserver + image: "{{ .Values.canary.image.path }}:{{ .Values.canary.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.canary.image.pullPolicy }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + ports: + - containerPort: 8270 + protocol: TCP + name: crabserver + - containerPort: 18270 + protocol: TCP + name: crabserver-mon + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- if .Values.canary.image.command }} + command: + {{- toYaml .Values.canary.image.command | nindent 10 }} + {{- end }} + {{- if .Values.canary.image.args }} + args: + {{- toYaml .Values.canary.image.args | nindent 10 }} + {{- end }} + env: + {{- toYaml .Values.canary.image.env | nindent 10 }} + livenessProbe: + {{- if eq (toString $environment) "prod" }} + {{- toYaml .Values.livenessProbe | nindent 12 }} + {{- else if eq (toString $environment) "preprod" }} + {{- toYaml .Values.livenessProbePreProd | nindent 12 }} + {{- else }} + {{- toYaml .Values.livenessProbeTest | nindent 12 }} + {{- end }} + readinessProbe: + {{- if eq (toString $environment) "prod" }} + {{- toYaml .Values.readinessProbe | nindent 12 }} + {{- else if eq (toString $environment) "preprod" }} + {{- toYaml .Values.readinessProbePreProd | nindent 12 }} + {{- else }} + {{- toYaml .Values.readinessProbeTest | nindent 12 }} + {{- end }} + volumeMounts: + - name: proxy-secrets + mountPath: /etc/proxy + readOnly: true + - name: secrets + mountPath: /etc/secrets + readOnly: true + - name: robot-secrets + mountPath: /etc/robots + readOnly: true + - name: hmac-secrets + mountPath: /etc/hmac + readOnly: true + - mountPath: /host/etc/grid-security + name: etc-grid-security + readOnly: true + - name: config + mountPath: /opt/config + - name: token-secrets + mountPath: /etc/token + readOnly: true + securityContext: + privileged: true + volumes: + - name: proxy-secrets + secret: + secretName: proxy-secrets + - name: secrets + secret: + secretName: crabserver-secrets + - name: robot-secrets + secret: + secretName: robot-secrets + - name: hmac-secrets + secret: + secretName: hmac-secrets + - name: etc-grid-security + hostPath: + path: /etc/grid-security + - name: config + configMap: + name: crabserver + - name: token-secrets + secret: + secretName: token-secrets +{{- end }} diff --git a/helm/crabserver/templates/deployment.yaml b/helm/crabserver/templates/deployment.yaml index af8a10644..478ca1ed3 100644 --- a/helm/crabserver/templates/deployment.yaml +++ b/helm/crabserver/templates/deployment.yaml @@ -1,4 +1,5 @@ {{- $environment := .Values.environment | default dict }} +{{- if .Values.enabled }} apiVersion: apps/v1 kind: Deployment metadata: @@ -92,8 +93,8 @@ spec: - mountPath: /host/etc/grid-security name: etc-grid-security readOnly: true - - name: setup-certs-and-run - mountPath: /opt/setup-certs-and-run + - name: config + mountPath: /opt/config - name: token-secrets mountPath: /etc/token readOnly: true @@ -115,9 +116,10 @@ spec: - name: etc-grid-security hostPath: path: /etc/grid-security - - name: setup-certs-and-run + - name: config configMap: name: crabserver - name: token-secrets secret: secretName: token-secrets +{{- end }} diff --git a/helm/crabserver/templates/filebeat-configmap.yaml b/helm/crabserver/templates/filebeat-configmap.yaml new file mode 100644 index 000000000..c84c6d60f --- /dev/null +++ b/helm/crabserver/templates/filebeat-configmap.yaml @@ -0,0 +1,37 @@ +{{- if .Values.logPipeline.enabled -}} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + k8s-app: filebeat + name: filebeat-crab-config + namespace: crab +data: + filebeat.yml: |- + filebeat.inputs: + - type: container + paths: + - /var/log/pods/crab_crabserver*/crabserver/*.log + include_lines: + - '\] (crabserver|crabserver-canary)-\w+-\w+ \d' + tags: + - "crabhttpcall" + - type: container + paths: + - /var/log/pods/crab_crabserver*/crabserver/*.log + include_lines: + - 'MeasureTime:seconds - ' + - 'MeasureSize:bytes - ' + tags: + - "crabrest" + output.logstash: + hosts: ["logstash-crab:5044"] + compression_level: 3 + bulk_max_size: 4096 + queue.mem: + events: 65536 + logging.metrics.enabled: false + logging.level: info + +{{- end }} diff --git a/helm/crabserver/templates/deployment-deamonset.yaml b/helm/crabserver/templates/filebeat-daemonset.yaml similarity index 74% rename from helm/crabserver/templates/deployment-deamonset.yaml rename to helm/crabserver/templates/filebeat-daemonset.yaml index 02a053c43..fa628947f 100644 --- a/helm/crabserver/templates/deployment-deamonset.yaml +++ b/helm/crabserver/templates/filebeat-daemonset.yaml @@ -1,3 +1,4 @@ +{{- if .Values.logPipeline.enabled -}} apiVersion: apps/v1 kind: DaemonSet metadata: @@ -20,14 +21,14 @@ spec: spec: containers: - args: - {{- toYaml .Values.deamonset.args | nindent 8 }} + {{- toYaml .Values.logPipeline.filebeat.args | nindent 8 }} env: - {{- toYaml .Values.deamonset.env | nindent 8 }} - image: {{ .Values.deamonset.path }} - imagePullPolicy: {{ .Values.deamonset.policy }} - name: {{ .Values.deamonset.name }} + {{- toYaml .Values.logPipeline.filebeat.env | nindent 8 }} + image: {{ .Values.logPipeline.filebeat.path }} + imagePullPolicy: {{ .Values.logPipeline.filebeat.policy }} + name: {{ .Values.logPipeline.filebeat.name }} resources: - {{- toYaml .Values.deamonset.resources | nindent 10 }} + {{- toYaml .Values.logPipeline.filebeat.resources | nindent 10 }} volumeMounts: - mountPath: /var/log/pods name: varlogpods @@ -59,3 +60,5 @@ spec: defaultMode: 416 name: filebeat-crab-config name: config + +{{- end }} diff --git a/helm/crabserver/templates/hpa.yaml b/helm/crabserver/templates/hpa.yaml index 81063b464..728361a56 100644 --- a/helm/crabserver/templates/hpa.yaml +++ b/helm/crabserver/templates/hpa.yaml @@ -1,4 +1,4 @@ -{{- if .Values.autoscaling.enabled }} +{{- if and (.Values.enabled) (.Values.autoscaling.enabled) }} apiVersion: autoscaling/v2beta1 kind: HorizontalPodAutoscaler metadata: diff --git a/helm/crabserver/templates/ingress.yaml b/helm/crabserver/templates/ingress.yaml index e34d0ad0a..5cb99ced2 100644 --- a/helm/crabserver/templates/ingress.yaml +++ b/helm/crabserver/templates/ingress.yaml @@ -1,4 +1,4 @@ -{{- if .Values.ingress.enabled -}} +{{- if and (.Values.enabled) (.Values.ingress.enabled) -}} {{- $fullName := include "crabserver.fullname" . -}} {{- $svcPort := .Values.service.port -}} {{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} diff --git a/helm/crabserver/templates/logstash-configmap.yaml b/helm/crabserver/templates/logstash-configmap.yaml new file mode 100644 index 000000000..a61c8e049 --- /dev/null +++ b/helm/crabserver/templates/logstash-configmap.yaml @@ -0,0 +1,29 @@ +{{- $environment := .Values.environment | default dict }} +{{- if .Values.logPipeline.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + k8s-app: logstash + name: logstash-crab +data: + # logstash config + logstash.yml: | +{{- if (eq (toString $environment) "prod") }} +{{ (.Files.Get "config/prod/logstash/logstash.yml" | indent 4) }} +{{- else if (eq (toString $environment) "preprod")}} +{{ (.Files.Get "config/preprod/logstash/logstash.yml" | indent 4) }} +{{- else }} +{{ (.Files.Get "config/test/logstash/logstash.yml" | indent 4) }} +{{- end }} + # pipeline config + logstash.conf: | +{{- if (eq (toString $environment) "prod") }} +{{ (.Files.Get "config/prod/logstash/logstash.conf" | indent 4) }} +{{- else if (eq (toString $environment) "preprod")}} +{{ (.Files.Get "config/preprod/logstash/logstash.conf" | indent 4) }} +{{- else }} +{{ (.Files.Get "config/test/logstash/logstash.conf" | indent 4) }} +{{- end }} + +{{- end }} diff --git a/kubernetes/cmsweb/monitoring/crab/logstash.yaml b/helm/crabserver/templates/logstash-deployment.yaml similarity index 56% rename from kubernetes/cmsweb/monitoring/crab/logstash.yaml rename to helm/crabserver/templates/logstash-deployment.yaml index 9ec3d1104..ad227f801 100644 --- a/kubernetes/cmsweb/monitoring/crab/logstash.yaml +++ b/helm/crabserver/templates/logstash-deployment.yaml @@ -1,29 +1,46 @@ +{{- $environment := .Values.environment | default dict }} +{{- if .Values.logPipeline.enabled -}} apiVersion: apps/v1 kind: Deployment metadata: - name: logstash - namespace: crab + name: logstash-crab labels: - app: logstash + app: logstash-crab spec: selector: matchLabels: - app: logstash + app: logstash-crab template: metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} labels: - app: logstash + app: logstash-crab spec: containers: - image: docker.elastic.co/logstash/logstash:8.1.3 name: logstash env: - name: CMSWEB_CLUSTER - value: dev # cluster + {{- if (eq (toString $environment) "prod") }} + value: prod + {{- else if (eq (toString $environment) "preprod")}} + value: testbed + {{- else }} + value: dev + {{- end }} - name: CMSWEB_ENV - value: dev # cmsweb_env + {{- if (eq (toString $environment) "prod") }} + value: prod + {{- else if (eq (toString $environment) "preprod")}} + value: testbed + {{- else }} + value: dev + {{- end }} - name: LS_JAVA_OPTS - value: "-Xms4g -Xmx4g" + value: "-Xms256m -Xmx4g" ports: - name: logstash containerPort: 5044 @@ -39,41 +56,27 @@ spec: args: - -f - /usr/share/logstash/pipeline/logstash.conf + - -e #- --log.level=debug # in k8s, output stdout prints only in debug resources: requests: - memory: 4Gi + memory: 512Mi cpu: "400m" limits: - memory: 8Gi + memory: 5Gi cpu: "4000m" volumes: - name: pipeline configMap: - name: logstash + name: logstash-crab items: - key: logstash.conf path: logstash.conf - name: config configMap: - name: logstash + name: logstash-crab items: - key: logstash.yml path: logstash.yml ---- -apiVersion: v1 -kind: Service -metadata: - name: logstash - namespace: crab - labels: - app: logstash -spec: - selector: - app: logstash - type: NodePort - ports: - - nodePort: 30044 - targetPort: 5044 - protocol: TCP - port: 5044 + +{{- end }} diff --git a/helm/crabserver/templates/logstash-service.yaml b/helm/crabserver/templates/logstash-service.yaml new file mode 100644 index 000000000..b754f2b09 --- /dev/null +++ b/helm/crabserver/templates/logstash-service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.logPipeline.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: logstash-crab + labels: + app: logstash-crab +spec: + selector: + app: logstash-crab + type: NodePort + ports: + - nodePort: 30044 + targetPort: 5044 + protocol: TCP + port: 5044 +{{- end }} diff --git a/helm/crabserver/templates/service.yaml b/helm/crabserver/templates/service.yaml index eb7acbbb3..8022584d8 100644 --- a/helm/crabserver/templates/service.yaml +++ b/helm/crabserver/templates/service.yaml @@ -1,3 +1,4 @@ +{{- if and (.Values.enabled) -}} kind: Service apiVersion: v1 metadata: @@ -13,4 +14,4 @@ spec: - port: 18270 targetPort: 18270 name: crabserver-mon ---- +{{- end }} diff --git a/helm/crabserver/templates/serviceaccount.yaml b/helm/crabserver/templates/serviceaccount.yaml index b3867549b..b0cd44582 100644 --- a/helm/crabserver/templates/serviceaccount.yaml +++ b/helm/crabserver/templates/serviceaccount.yaml @@ -1,4 +1,4 @@ -{{- if .Values.serviceAccount.create -}} +{{- if and (.Values.enabled) (.Values.serviceAccount.create) -}} apiVersion: v1 kind: ServiceAccount metadata: diff --git a/helm/crabserver/values-canary.yaml b/helm/crabserver/values-canary.yaml new file mode 100644 index 000000000..b6439ae19 --- /dev/null +++ b/helm/crabserver/values-canary.yaml @@ -0,0 +1,19 @@ +environment: "test" + +image: + tag: "pypi-test12-manual1" + +canary: + enabled: true + replicaCount: 1 + image: + tag: "pypi-test12-manual1" + +# disable liveness/readiness +# https://helm.sh/docs/chart_template_guide/values_files/#deleting-a-default-key +livenessProbePreProd: null +readinessProbePreProd: null +livenessProbeTest: null +readinessProbeTest: null +livenessProbe: null +readinessProbe: null diff --git a/helm/crabserver/values-genall.yaml b/helm/crabserver/values-genall.yaml new file mode 100644 index 000000000..663067d1c --- /dev/null +++ b/helm/crabserver/values-genall.yaml @@ -0,0 +1,17 @@ +--- +enabled: true +environment: "test" + +crabserver: + image: + tag: "pypi-test12-manual1" + canary: + enabled: true + replicaCount: 1 + image: + tag: "pypi-test12-manual1" + +ciServiceAccount: + enabled: true +logPipeline: + enabled: true diff --git a/helm/crabserver/values-logpipeline.yaml b/helm/crabserver/values-logpipeline.yaml new file mode 100644 index 000000000..4924616eb --- /dev/null +++ b/helm/crabserver/values-logpipeline.yaml @@ -0,0 +1,6 @@ +environment: "test" + +enabled: false + +logPipeline: + enabled: true diff --git a/helm/crabserver/values-preprod-pypi.yaml b/helm/crabserver/values-preprod-pypi.yaml deleted file mode 100644 index 2855a4ce8..000000000 --- a/helm/crabserver/values-preprod-pypi.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -environment: "preprod" - -image: - path: registry.cern.ch/cmscrab/crabserver - pullPolicy: IfNotPresent - tag: "v3.240731-stable" - command: - - /data/entrypoint.sh - args: - - /bin/bash - - -c - - | - sudo cp -r /host/etc/grid-security/* /etc/grid-security \ - && echo 'INFO Files in /etc/grid-security' \ - && ls -lahZ /etc/grid-security \ - && /data/run.sh - diff --git a/helm/crabserver/values-prod-pypi.yaml b/helm/crabserver/values-prod-pypi.yaml deleted file mode 100644 index 30a862560..000000000 --- a/helm/crabserver/values-prod-pypi.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -environment: "prod" - -image: - path: registry.cern.ch/cmscrab/crabserver - pullPolicy: IfNotPresent - tag: "v3.240731-stable" - command: - - /data/entrypoint.sh - args: - - /bin/bash - - -c - - | - sudo cp -r /host/etc/grid-security/* /etc/grid-security \ - && echo 'INFO Files in /etc/grid-security' \ - && ls -lahZ /etc/grid-security \ - && /data/run.sh - diff --git a/helm/crabserver/values-prod.yaml b/helm/crabserver/values-prod.yaml index 8a358df5c..ca34fd200 100644 --- a/helm/crabserver/values-prod.yaml +++ b/helm/crabserver/values-prod.yaml @@ -1,5 +1,4 @@ environment: "prod" image: - tag: "v3.240731" - path: "registry.cern.ch/cmsweb/crabserver" + tag: "v3.240809-stable" diff --git a/helm/crabserver/values-test11-pypi.yaml b/helm/crabserver/values-test11-pypi.yaml deleted file mode 100644 index d5c69bdf7..000000000 --- a/helm/crabserver/values-test11-pypi.yaml +++ /dev/null @@ -1,23 +0,0 @@ ---- -environment: "test" - -image: - path: registry.cern.ch/cmscrab/crabserver - pullPolicy: IfNotPresent - tag: "pypi-test2-1716890482" - command: - - /data/entrypoint.sh - args: - - /bin/bash - - -c - - | - sudo cp -r /host/etc/grid-security/* /etc/grid-security \ - && echo 'INFO Files in /etc/grid-security' \ - && ls -lahZ /etc/grid-security \ - && /data/run.sh - -#https://helm.sh/docs/chart_template_guide/values_files/#deleting-a-default-key -livenessProbePreProd: null -readinessProbePreProd: null -readinessProbe: null -livenessProbe: null diff --git a/helm/crabserver/values-test12.yaml b/helm/crabserver/values-test12.yaml index 60ace5453..cec1c9875 100644 --- a/helm/crabserver/values-test12.yaml +++ b/helm/crabserver/values-test12.yaml @@ -2,7 +2,7 @@ environment: "test" image: - tag: "v3.240530" + tag: "pypi-test12-manual1" # disable liveness/readiness # https://helm.sh/docs/chart_template_guide/values_files/#deleting-a-default-key diff --git a/helm/crabserver/values-test1.yaml b/helm/crabserver/values-test14.yaml similarity index 100% rename from helm/crabserver/values-test1.yaml rename to helm/crabserver/values-test14.yaml diff --git a/helm/crabserver/values-testbed-pypi.yaml b/helm/crabserver/values-testbed-pypi.yaml deleted file mode 100644 index 8b8e2e8b5..000000000 --- a/helm/crabserver/values-testbed-pypi.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -environment: "testbed" - -image: - path: registry.cern.ch/cmscrab/crabserver - pullPolicy: IfNotPresent - tag: "v3.240731-stable" - command: - - /data/entrypoint.sh - args: - - /bin/bash - - -c - - | - sudo cp -r /host/etc/grid-security/* /etc/grid-security \ - && echo 'INFO Files in /etc/grid-security' \ - && ls -lahZ /etc/grid-security \ - && /data/run.sh - diff --git a/helm/crabserver/values-testbed.yaml b/helm/crabserver/values-testbed.yaml index 4abdbc172..440bc22fd 100644 --- a/helm/crabserver/values-testbed.yaml +++ b/helm/crabserver/values-testbed.yaml @@ -1,4 +1,4 @@ environment: "preprod" image: - tag: "v3.240515" + tag: "v3.240809-stable" diff --git a/helm/crabserver/values.yaml b/helm/crabserver/values.yaml index a210351c7..d43982a92 100644 --- a/helm/crabserver/values.yaml +++ b/helm/crabserver/values.yaml @@ -2,25 +2,43 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. -replicaCount: +enabled: true + +replicaCount: prod: 15 preprod: 1 test: 1 image: path: registry.cern.ch/cmscrab/crabserver - pullPolicy: Always + pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. tag: "v3.231006" command: - /bin/bash - - /opt/setup-certs-and-run/setup-certs-and-run.sh + - /opt/config/setup.sh env: - name: CRABSERVER_LOGSTDOUT value: "t" - name: CRABSERVER_THREAD_POOL value: "2" +canary: + enabled: false + image: + path: registry.cern.ch/cmscrab/crabserver + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "v3.231006" + command: + - /bin/bash + - /opt/config/setup.sh + env: + - name: CRABSERVER_LOGSTDOUT + value: "t" + - name: CRABSERVER_THREAD_POOL + value: "2" + environment: imagePullSecrets: [] @@ -36,7 +54,7 @@ serviceAccount: # If not set and create is true, a name is generated using the fullname template name: "" -podAnnotations: +podAnnotations: prometheus.io/scrape: 'true' prometheus.io/port: "18270" @@ -48,7 +66,7 @@ podSecurityContext: imageFilebeatSecurityContext: allowPrivilegeEscalation: false -securityContext: +securityContext: privileged: true serviceMon: @@ -62,7 +80,7 @@ ingress: # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" -resources: +resources: limits: cpu: 1500m memory: 3Gi @@ -70,38 +88,6 @@ resources: cpu: 200m memory: 256Mi - -deamonset: - name: filebeat - path: docker.elastic.co/beats/filebeat:8.5.1 - policy: IfNotPresent - args: - - bash - - -c - - filebeat -c /etc/filebeat.yml --path.data /data/filebeat/${MY_NODE_NAME}/data -e - env: - - name: MY_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: MY_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - resources: - requests: - memory: "128Mi" - cpu: "200m" - limits: - cpu: "1" - memory: "1Gi" - - livenessProbe: exec: command: @@ -121,7 +107,7 @@ readinessProbe: - -c - | cmsweb-ping --url=http://localhost:8270/crabserver/prod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]" - periodSeconds: 60 + periodSeconds: 60 timeoutSeconds: 60 livenessProbePreProd: @@ -143,7 +129,7 @@ readinessProbePreProd: - -c - | cmsweb-ping --url=http://localhost:8270/crabserver/preprod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]" - periodSeconds: 60 + periodSeconds: 60 timeoutSeconds: 60 livenessProbeTest: @@ -165,7 +151,7 @@ readinessProbeTest: - -c - | cmsweb-ping --url=http://localhost:8270/crabserver/dev/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]" - periodSeconds: 60 + periodSeconds: 60 timeoutSeconds: 60 autoscaling: @@ -180,3 +166,39 @@ nodeSelector: {} tolerations: [] affinity: {} + + +ciServiceAccount: + enabled: false + +logPipeline: + enabled: false + filebeat: + name: filebeat + path: docker.elastic.co/beats/filebeat:8.5.1 + policy: IfNotPresent + args: + - bash + - -c + - filebeat -c /etc/filebeat.yml --path.data /data/filebeat/${MY_NODE_NAME}/data -e + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + resources: + requests: + memory: "128Mi" + cpu: "200m" + limits: + cpu: "1" + memory: "1Gi" diff --git a/kubernetes/cmsweb/monitoring/crab/README.md b/kubernetes/cmsweb/monitoring/crab/README.md deleted file mode 100644 index 86594b6da..000000000 --- a/kubernetes/cmsweb/monitoring/crab/README.md +++ /dev/null @@ -1,53 +0,0 @@ -## CRAB LOGSTASH - -Used for parsing CRAB Taskworker VM instance logs. Filebeat should send data with `crabtaskworker` tag. - -- Refs: https://monit-docs.web.cern.ch/metrics/http/ -- https://its.cern.ch/jira/browse/CMSMONIT-460 - -#### Resorved keywords - -They are used in grok parsers (alpha order) !!please update the list when you update the grok!! - -``` -- acquiredFiles, acquiredFilesStatus, action, -- blocks, -- completionTime, -- exceptionHandled, -- files, filesPublished, functionName, -- logMsg, log_type, -- publicationResult, publisher_json_data, -- slaveID, -- taskID, taskName, timestamp_temp, tw_json_data, -- Worker, workType, -``` - -#### Example log lines for grok - -There are currently 10 different grok definitions. Example log lines for each **log_type**: - -- `work_on_task_completed` - - `2020-10-21 00:11:41,973:DEBUG:Worker,111:Process-6: KILL work on 201020_214355:mimacken_crab_2017_LFVAnalysis_SingleEle_F completed in 2 seconds: Status: OK` -- `publisher_config_data` - - `2021-05-19 09:15:34,006:INFO:PublisherMaster,149:PUBSTART: {"max_slaves": 5, "dryRun": false, "asoworker": "schedd", "DBShost": "cmsweb-prod.cern.ch", "instance": "test2", "version": "v3.210514"}` -- `tw_config_data` - - `2021-05-25 18:50:55,460:INFO:MasterWorker,174:TWSTART: {"restHost": "X", "name": "Y", "recurringActions": ["RemovetmpDir", "BanDestinationSites", "TapeRecallStatus"], "DBSHostName": "Z", "instance": "other", "version": "development", "dbInstance": "dev", "nslaves": 1}` -- `start_new_task` - - `2020-09-10 04:56:49,748:DEBUG:Worker,104:Process-6: Starting on 200901_130305:wjang_crab_NanoAODv7_v0_QCD_HT2000toInf_TuneCUETP8M1_13TeV-madgraphMLM-pythia8` -- `failed_publication` - - `2020-11-21 01:45:32,789:ERROR:PublisherMaster,550:Taskname 201118_182833:vcepaiti_crab_QCD_Pt-80to120_EMEnriched_TuneCUETP8M1_13TeV_pythia8-2016_NANOX_201117 : 1 blocks failed for a total of 11 files` -- `successful_publication` - - `2020-11-27 14:25:16,191:INFO:PublisherMaster,545:Taskname 201127_011713:anstahll_crab_AODSkim_HIMinimumBias14_HIRun2018_04Apr2019_DiMuMassMin2_20201117 is OK. Published 37 files in 1 blocks.` -- `publication_error` - - `2020-11-26 19:23:27,737:ERROR:PublisherMaster,554:Exception when calling TaskPublish!` -- `acquired_files` - - `2021-04-15 22:23:39,566:DEBUG:master: 8 : 210415_093249:algomez_crab_QCDHT100to200TuneCP5PSWeights13TeV-madgraphMLM` -- `acquired_files_status` - - `2021-12-03 20:13:59,965:DEBUG:PublisherMaster,413:acquired_files: OK 89 : 211203_174945:cmsbot_crab_20211203_184942` -- `action_on_task_finished` - - `2020-09-10 04:48:50,091:INFO:Handler,104:Finished on 200901_125412:wjang_crab_NanoAODv7_v0_QCD_HT700to1000_TuneCUETP8M1_13TeV-madgraphMLM-pythia8 in 1 seconds` - - -#### Notes - -Tested with current filebeat version in VM: 7.10.0 From 29a4e162f8ba1b0ec46a06a0720c7c7b22b76777 Mon Sep 17 00:00:00 2001 From: Thanayut Seethongchuen Date: Mon, 2 Sep 2024 16:48:28 +0200 Subject: [PATCH 2/6] fix deploy.sh script --- helm/crabserver/deploy.sh | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/helm/crabserver/deploy.sh b/helm/crabserver/deploy.sh index 747d15c44..6d0f02945 100755 --- a/helm/crabserver/deploy.sh +++ b/helm/crabserver/deploy.sh @@ -1,7 +1,7 @@ #! /bin/bash set -euo pipefail -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +ENVNAME=$1 # hash table of clusters nickname-name: declare -A cluster_map=([prod]=prod) @@ -11,20 +11,31 @@ cluster_map[testbed]=testbed cluster_map[test2]=test2 cluster_map[test11]=test11 cluster_map[test12]=test12 +cluster_map[test14]=test14 + +declare -A valuefile_map=([prod]=prod) +valuefile_map[prod]=values-prod.yaml +valuefile_map[preprod]=values-preprod.yaml +valuefile_map[testbed]=values-testbed.yaml +valuefile_map[test2]=values-testx.yaml +valuefile_map[test11]=values-testx.yaml +valuefile_map[test12]=values-testx.yaml +valuefile_map[test14]=values-testx.yaml + if [[ $# -ne 1 ]]; then echo "Usage: deploy.sh ENVNAME" - echo " ENVNAME=(prod|preprod|testbed|test2|test11|test12)" + echo " ENVNAME=(prod|preprod|testbed|test2|test11|test12|test14)" exit 1 fi -desired_cluster="${cluster_map[$1]}" +desired_cluster="${cluster_map[$ENVNAME]}" +valuefile="${valuefile_map[$ENVNAME]}" # make sure that your current context points to the desired cluster current_cluster=$(kubectl config view -o json | jq '.["current-context"] as $context | .["contexts"][] | select(.name | contains($context))| .context.cluster') - -if [[ $current_cluster =~ $desired_cluster ]]; then - echo "deploying to $desired_cluster"; - helm template crabserver . -f values.yaml -f values-${1}-pypi.yaml | kubectl -n crab apply -f - -else - echo "wrong cluster: your are connected to $current_cluster"; +set -x +if [[ $current_cluster =~ $desired_cluster ]]; then + echo "deploying to $desired_cluster"; + helm template crabserver . -f values.yaml -f "${valuefile}" | kubectl -n crab apply -f - +else + echo "Error: wrong cluster. your are connected to $current_cluster"; fi - From 4b759fd412595f5888d6dee624494bf3481f8302 Mon Sep 17 00:00:00 2001 From: Thanayut Seethongchuen Date: Mon, 2 Sep 2024 16:51:06 +0200 Subject: [PATCH 3/6] modify canary and unify values of test cluster --- helm/crabserver/values-canary.yaml | 13 ++----------- helm/crabserver/values-test11.yaml | 14 -------------- helm/crabserver/values-test14.yaml | 4 ---- helm/crabserver/values-test2.yaml | 4 ---- ...alues-test12.yaml => values-testx-noprobe.yaml} | 2 +- helm/crabserver/values-testx.yaml | 5 +++++ 6 files changed, 8 insertions(+), 34 deletions(-) delete mode 100644 helm/crabserver/values-test11.yaml delete mode 100644 helm/crabserver/values-test14.yaml delete mode 100644 helm/crabserver/values-test2.yaml rename helm/crabserver/{values-test12.yaml => values-testx-noprobe.yaml} (90%) create mode 100644 helm/crabserver/values-testx.yaml diff --git a/helm/crabserver/values-canary.yaml b/helm/crabserver/values-canary.yaml index b6439ae19..8658921e8 100644 --- a/helm/crabserver/values-canary.yaml +++ b/helm/crabserver/values-canary.yaml @@ -1,19 +1,10 @@ environment: "test" image: - tag: "pypi-test12-manual1" + tag: "v3.240809-stable" canary: enabled: true replicaCount: 1 image: - tag: "pypi-test12-manual1" - -# disable liveness/readiness -# https://helm.sh/docs/chart_template_guide/values_files/#deleting-a-default-key -livenessProbePreProd: null -readinessProbePreProd: null -livenessProbeTest: null -readinessProbeTest: null -livenessProbe: null -readinessProbe: null + tag: "v3.240809-stable" diff --git a/helm/crabserver/values-test11.yaml b/helm/crabserver/values-test11.yaml deleted file mode 100644 index 1c3ab3927..000000000 --- a/helm/crabserver/values-test11.yaml +++ /dev/null @@ -1,14 +0,0 @@ -environment: "test" - -image: - tag: "v3.240515" - -livenessProbeTest: - failureThreshold: 3 - initialDelaySeconds: 120 - periodSeconds: 600 - timeoutSeconds: 60 - -readinessProbeTest: - periodSeconds: 600 - timeoutSeconds: 60 diff --git a/helm/crabserver/values-test14.yaml b/helm/crabserver/values-test14.yaml deleted file mode 100644 index 2df6ae869..000000000 --- a/helm/crabserver/values-test14.yaml +++ /dev/null @@ -1,4 +0,0 @@ -environment: "test" - -image: - tag: "v3.240709" diff --git a/helm/crabserver/values-test2.yaml b/helm/crabserver/values-test2.yaml deleted file mode 100644 index 6e11536f4..000000000 --- a/helm/crabserver/values-test2.yaml +++ /dev/null @@ -1,4 +0,0 @@ -environment: "test" - -image: - tag: "v3.240515" diff --git a/helm/crabserver/values-test12.yaml b/helm/crabserver/values-testx-noprobe.yaml similarity index 90% rename from helm/crabserver/values-test12.yaml rename to helm/crabserver/values-testx-noprobe.yaml index cec1c9875..6f16d2302 100644 --- a/helm/crabserver/values-test12.yaml +++ b/helm/crabserver/values-testx-noprobe.yaml @@ -2,7 +2,7 @@ environment: "test" image: - tag: "pypi-test12-manual1" + tag: "v3.240809-stable" # disable liveness/readiness # https://helm.sh/docs/chart_template_guide/values_files/#deleting-a-default-key diff --git a/helm/crabserver/values-testx.yaml b/helm/crabserver/values-testx.yaml new file mode 100644 index 000000000..163272844 --- /dev/null +++ b/helm/crabserver/values-testx.yaml @@ -0,0 +1,5 @@ +--- +environment: "test" + +image: + tag: "v3.240809-stable" From 7d0965701c7f1fdbeb25f49ca1f209757dc02695 Mon Sep 17 00:00:00 2001 From: Thanayut Seethongchuen Date: Wed, 4 Sep 2024 20:54:54 +0200 Subject: [PATCH 4/6] rename setup.sh back to entrypoint.sh --- helm/crabserver/scripts/{setup.sh => entrypoint.sh} | 6 +++--- helm/crabserver/templates/configMap.yaml | 4 ++-- helm/crabserver/values.yaml | 8 ++++++-- 3 files changed, 11 insertions(+), 7 deletions(-) rename helm/crabserver/scripts/{setup.sh => entrypoint.sh} (89%) diff --git a/helm/crabserver/scripts/setup.sh b/helm/crabserver/scripts/entrypoint.sh similarity index 89% rename from helm/crabserver/scripts/setup.sh rename to helm/crabserver/scripts/entrypoint.sh index 995bace70..2c0600dc4 100644 --- a/helm/crabserver/scripts/setup.sh +++ b/helm/crabserver/scripts/entrypoint.sh @@ -21,8 +21,8 @@ sudo chown $USER:$USER /data/srv/current/config/crabserver/config.py sudo cp /etc/secrets/CRABServerAuth.py /data/srv/current/auth/crabserver/CRABServerAuth.py sudo chown $USER:$USER /data/srv/current/auth/crabserver/CRABServerAuth.py -# test X509_USER_PROXY -# Wa: I do not understand why we use this file instead of robotcert +# export proxy cert generated by cron-proxy cronjob +# test if file exist ls /etc/proxy/proxy export X509_USER_PROXY=/etc/proxy/proxy @@ -31,4 +31,4 @@ sudo cp /host/etc/grid-security/* /etc/grid-security echo 'INFO Files in /etc/grid-security' ls -lahZ /etc/grid-security -exec /usr/bin/tini -- /data/run.sh +exec /usr/bin/tini -- "$@" diff --git a/helm/crabserver/templates/configMap.yaml b/helm/crabserver/templates/configMap.yaml index cdbfc4a4c..3ddc639d1 100644 --- a/helm/crabserver/templates/configMap.yaml +++ b/helm/crabserver/templates/configMap.yaml @@ -8,8 +8,8 @@ metadata: app: crabserver namespace: crab data: - setup.sh: | -{{ (.Files.Get "scripts/setup.sh" | indent 4) }} + entrypoint.sh: | +{{ (.Files.Get "scripts/entrypoint.sh" | indent 4) }} config.py: | {{- if (eq (toString $environment) "prod") }} {{ (.Files.Get "config/prod/config.py" | indent 4) }} diff --git a/helm/crabserver/values.yaml b/helm/crabserver/values.yaml index d43982a92..dc3378b4a 100644 --- a/helm/crabserver/values.yaml +++ b/helm/crabserver/values.yaml @@ -16,7 +16,9 @@ image: tag: "v3.231006" command: - /bin/bash - - /opt/config/setup.sh + - /opt/config/entrypoint.sh + args: + - /data/run.sh env: - name: CRABSERVER_LOGSTDOUT value: "t" @@ -32,7 +34,9 @@ canary: tag: "v3.231006" command: - /bin/bash - - /opt/config/setup.sh + - /opt/config/entrypoint.sh + args: + - /data/run.sh env: - name: CRABSERVER_LOGSTDOUT value: "t" From 3703f4e8260d379013e5c3abb06c5aa0c50f4f22 Mon Sep 17 00:00:00 2001 From: Thanayut Seethongchuen Date: Fri, 13 Sep 2024 18:43:33 +0200 Subject: [PATCH 5/6] update README.md, add some comments, bump prod version --- helm/crabserver/README.md | 68 +++++++++++++++++++----- helm/crabserver/values-canary.yaml | 8 +-- helm/crabserver/values-clusteradmin.yaml | 14 +++++ helm/crabserver/values-genall.yaml | 1 + helm/crabserver/values-prod.yaml | 2 +- helm/crabserver/values-testbed.yaml | 2 +- 6 files changed, 78 insertions(+), 17 deletions(-) create mode 100644 helm/crabserver/values-clusteradmin.yaml diff --git a/helm/crabserver/README.md b/helm/crabserver/README.md index 805314299..8f2b55ba7 100644 --- a/helm/crabserver/README.md +++ b/helm/crabserver/README.md @@ -1,19 +1,63 @@ -## Deploy CRAB +# Deploy CRAB -At the moment CRAB team is transitioning to using helm. However, we do noy like -to use helm for deployment. See this -[comment](https://github.com/dmwm/CRABServer/issues/7843#issuecomment-2025085120) -for some context. +In this helm chart, we separate k8s resources into 3 resource groups: -We like to use helm for templating, using the helm charts to generate the -manifest file, then applying it with `kubectl apply`. +1. `crabserver` manifest for CRAB REST, including `crabserver-canary`. +2. `logPipline`, the log pipeline which have logstash, filebeat and it config. +3. `ciServiceAccount` with deployment permission, for deploying new image from CI. Note that only cluster admin could not `rolebinding` in testbed and production cluster. -We therefore ask cmsweb operators to deploy crab following the procedure at -[1]. +CRAB team will only handle the first resources group, `crabserver`, to deploy a new service version or modify how we start our service. The rest we kindly ask cluster admin to take care of. -Changes to `../../kubernetes/cmsweb/services/crabserver.yaml` should not be committed. +## Configuration (helm's value) ---- +There are helm value, `enabled`, that is used to control which resource group will be generated. -[1] https://cmscrab.docs.cern.ch/technical/crab-rest/deploy.html#deploy-on-kubernetes-use-helm-template-to-generate-manifest-preferred +#### enabled +Default: `true` + +Generate `crabserver` resources group. + +#### canary.enabled + +Default: `false` + +Enable canary deployment, a.k.a. the `crabserver-canary` Deployment manifest. + +#### logPipeline:enabled + +Default: `false` + +Generate `logPipeline` resources group. + +#### ciServiceAccount.enabled + +Generate `ciServiceAccount` resources group. Need cluster admin for applying `role` and `rolebinding`. + +## Deploy + +### For cluster admins + +To deploy everything on new testbed and production cluster: + +1. Deploy `crabserver-secrets` credential. + +2. Deploy all services: + ``` + helm install crabserver . -f values.yaml -f values-clusteradmin.yaml --set environment=preprod + ``` + Please change `environment=preprod` to the cluster you are deploying (`preprod` for testbed and preprod cluster, `prod` for production cluster). + +Please leave test cluster to CRAB operators. + +### For CRAB operators + +Please consult [Deploying CRAB REST](https://cmscrab.docs.cern.ch/crab-components/crab-rest/deploy.html). + +## Regarding deploy with `helm install` command + +CRAB team does not use helm for deployment. We like to use helm for templating, using the helm charts to generate the manifest file, then applying it with `kubectl apply`. See this [comment](https://github.com/dmwm/CRABServer/issues/7843#issuecomment-2025085120) for some context. + +However, it should not have any conflict if cluster admins run `helm install` **before** CRAB operators apply new manifests with `kubectl apply`. + +In case of conflict, feel free to purge all resources and reinstall with helm again. diff --git a/helm/crabserver/values-canary.yaml b/helm/crabserver/values-canary.yaml index 8658921e8..7716ca584 100644 --- a/helm/crabserver/values-canary.yaml +++ b/helm/crabserver/values-canary.yaml @@ -1,10 +1,12 @@ -environment: "test" +--- +# Deploying canary together with normal deployment +environment: "preprod" image: - tag: "v3.240809-stable" + tag: "v3.240904-stable" canary: enabled: true replicaCount: 1 image: - tag: "v3.240809-stable" + tag: "v3.240904-stable" diff --git a/helm/crabserver/values-clusteradmin.yaml b/helm/crabserver/values-clusteradmin.yaml new file mode 100644 index 000000000..2890bfedf --- /dev/null +++ b/helm/crabserver/values-clusteradmin.yaml @@ -0,0 +1,14 @@ +--- +enabled: true +environment: "preprod" + +crabserver: + image: + tag: "v3.240904-stable" + canary: + enabled: false + +ciServiceAccount: + enabled: true +logPipeline: + enabled: true diff --git a/helm/crabserver/values-genall.yaml b/helm/crabserver/values-genall.yaml index 663067d1c..e8aaa1350 100644 --- a/helm/crabserver/values-genall.yaml +++ b/helm/crabserver/values-genall.yaml @@ -1,4 +1,5 @@ --- +# For debugging only. Generate all resources in this chart. enabled: true environment: "test" diff --git a/helm/crabserver/values-prod.yaml b/helm/crabserver/values-prod.yaml index ca34fd200..f63764119 100644 --- a/helm/crabserver/values-prod.yaml +++ b/helm/crabserver/values-prod.yaml @@ -1,4 +1,4 @@ environment: "prod" image: - tag: "v3.240809-stable" + tag: "v3.240904-stable" diff --git a/helm/crabserver/values-testbed.yaml b/helm/crabserver/values-testbed.yaml index 440bc22fd..64a304e6f 100644 --- a/helm/crabserver/values-testbed.yaml +++ b/helm/crabserver/values-testbed.yaml @@ -1,4 +1,4 @@ environment: "preprod" image: - tag: "v3.240809-stable" + tag: "v3.240904-stable" From 50014775387eb70d7a1de74e1246ecd2af73079b Mon Sep 17 00:00:00 2001 From: Thanayut Seethongchuen Date: Fri, 13 Sep 2024 18:46:34 +0200 Subject: [PATCH 6/6] some typo --- helm/crabserver/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/crabserver/README.md b/helm/crabserver/README.md index 8f2b55ba7..73a31e11d 100644 --- a/helm/crabserver/README.md +++ b/helm/crabserver/README.md @@ -24,7 +24,7 @@ Default: `false` Enable canary deployment, a.k.a. the `crabserver-canary` Deployment manifest. -#### logPipeline:enabled +#### logPipeline.enabled Default: `false` @@ -32,7 +32,7 @@ Generate `logPipeline` resources group. #### ciServiceAccount.enabled -Generate `ciServiceAccount` resources group. Need cluster admin for applying `role` and `rolebinding`. +Generate `ciServiceAccount` resources group. Need cluster admin for applying `role` and `rolebinding` of k8s service account. ## Deploy