From 1ed4d71bf1068cae4a36236f2531583afbf0b8cb Mon Sep 17 00:00:00 2001 From: Stefano Belforte Date: Fri, 12 Nov 2021 00:48:07 +0100 Subject: [PATCH] Port to python3 recent small fixes from master (#6858) * use gocurl from CVMFS Fix #6822 (#6823) * add comment about py2/3 compatibility needs (#6826) * add GH remote for Diego * upload new config version (#6852) * stop trying to remove failed migrations from 2019. Fix #6854 (#6855) Co-authored-by: Daina <60326190+ddaina@users.noreply.github.com> --- Docker/addGH.sh | 2 + .../Monitor/logstash/crabtaskworker.conf | 183 +++++++++++++++++- 2 files changed, 182 insertions(+), 3 deletions(-) diff --git a/Docker/addGH.sh b/Docker/addGH.sh index cbf94377de..8f329cbf50 100755 --- a/Docker/addGH.sh +++ b/Docker/addGH.sh @@ -23,12 +23,14 @@ git checkout $CRABServerTag git remote add stefano https://github.com/belforte/CRABServer.git git remote add daina https://github.com/ddaina/CRABServer.git git remote add mapellidario https://github.com/mapellidario/CRABServer.git +git remote add diego https://github.com/dciangot/CRABServer.git cd .. cd WMCore git checkout $WMCoreTag git remote add stefano https://github.com/belforte/WMCore.git git remote add daina https://github.com/ddaina/WMCore.git git remote add mapellidario https://github.com/mapellidario/WMCore.git +git remote add diego https://github.com/dciangot/WMCore.git # 4. create .gitconfig cat > ~/.gitconfig << EOF diff --git a/src/script/Monitor/logstash/crabtaskworker.conf b/src/script/Monitor/logstash/crabtaskworker.conf index 9f8345fd8d..64f3895240 100644 --- a/src/script/Monitor/logstash/crabtaskworker.conf +++ b/src/script/Monitor/logstash/crabtaskworker.conf @@ -5,6 +5,13 @@ input { } filter{ + + mutate { add_field => { + "hostname" => "%{[cloud][instance][name]}" + "log_file" => "%{[log][file][path]}" + } + } + if [message] =~ /.*completed in.*/ { grok{ match => { @@ -14,6 +21,34 @@ filter{ add_field => {"log_type" => "work_on_task_completed"} overwrite => ["message"] } + + +###BELOW + } else if [message] =~ /.*PUBSTART.*/ { + grok{ + match => { + #2021-05-19 09:15:34,006:INFO:PublisherMaster,149:PUBSTART: {"max_slaves": 5, "dryRun": false, "asoworker": "schedd", "DBShost": "cmsweb-prod.cern.ch", "instance": "test2", "version": "v3.210514"} + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:PUBSTART:%{GREEDYDATA:publisher_json_data}" + } + add_field => {"log_type" => "publisher_config_data"} + overwrite => ["message"] + } +###END + + +###BELOW + } else if [message] =~ /.*TWSTART.*/ { + grok{ + match => { + #2021-05-25 18:50:55,460:INFO:MasterWorker,174:TWSTART: {"restHost": "X", "name": "Y", "recurringActions": ["RemovetmpDir", "BanDestinationSites", "TapeRecallStatus"], "DBSHostName": "Z", "instance": "other", "version": "development", "dbInstance": "dev", "nslaves": 1} + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:MasterWorker,%{INT}:TWSTART: %{GREEDYDATA:tw_json_data}" + } + add_field => {"log_type" => "tw_config_data"} + overwrite => ["message"] + } +###END + + } else if [message] =~ /.*Starting.*at.*on.*/ { grok{ match => { @@ -23,6 +58,47 @@ filter{ add_field => {"log_type" => "start_new_task"} overwrite => ["message"] } + + } else if [message] =~ /.*blocks failed.*files.*/ { + grok{ + match => { + #2020-11-21 01:45:32,789:ERROR:PublisherMaster,550:Taskname 201118_182833:vcepaiti_crab_QCD_Pt-80to120_EMEnriched_TuneCUETP8M1_13TeV_pythia8-2016_NANOX_201117 : 1 blocks failed for a total of 11 files + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:Taskname %{NOTSPACE:taskName} : %{INT:blocks} blocks failed for a total of %{INT:files} files" + } + add_field => {"log_type" => "failed_publication"} + overwrite => ["message"] + } + + } else if [message] =~ /.*Published.*files in.*/ { + grok{ + match => { + #2020-11-27 14:25:16,191:INFO:PublisherMaster,545:Taskname 201127_011713:anstahll_crab_AODSkim_HIMinimumBias14_HIRun2018_04Apr2019_DiMuMassMin2_20201117 is OK. Published 37 files in 1 blocks. + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:Taskname %{NOTSPACE:taskName} is %{NOTSPACE:publicationResult}. Published %{INT:filesPublished} files in %{INT:blocks} blocks." + } + add_field => {"log_type" => "successful_publication"} + overwrite => ["message"] + } + + } else if [message] =~ /.*Exception.*TaskPublish.*/ { + grok{ + match => { + #2020-11-26 19:23:27,737:ERROR:PublisherMaster,554:Exception when calling TaskPublish! + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:%{GREEDYDATA:exceptionHandled}" + } + add_field => {"log_type" => "publication_error"} + overwrite => ["message"] + } + + } else if [message] =~ /.*DEBUG:master.*/ { + grok{ + match => { + #2021-04-15 22:23:39,566:DEBUG:master: 8 : 210415_093249:algomez_crab_QCDHT100to200TuneCP5PSWeights13TeV-madgraphMLM + "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:master:%{SPACE}%{INT:acquiredFiles}%{SPACE}:%{SPACE}%{NOTSPACE:taskName}" + } + add_field => {"log_type" => "acquired_files"} + overwrite => ["message"] + } + } else if [message] =~ /.*Finished.*object at .*/ { grok{ match => { @@ -40,10 +116,42 @@ filter{ } ruby { code => "event.set('timestamp', (event.get('timestamp_str').to_f * 1000).to_i)" } ruby { code => "event.set('producer_time', (event.get('@timestamp').to_f * 1000).to_i)" } + ruby { code => "event.set('rec_timestamp', (event.get('timestamp_str').to_f * 1000).to_i)" } + + if [publisher_json_data] { + json{ + source => "publisher_json_data" + target => "publisher_json_data" + } + + ruby { + code => ' + event.get("publisher_json_data").each { |k, v| + event.set(k,v) + } + event.remove("publisher_json_data") + ' + } + } else if [tw_json_data] { + json{ + source => "tw_json_data" + target => "tw_json_data" + } + + ruby { + code => ' + event.get("tw_json_data").each { |k, v| + event.set(k,v) + } + event.remove("tw_json_data") + ' + } + } } + output { if [log_type] == "work_on_task_completed" { http { @@ -52,7 +160,7 @@ output { #url => "http://localhost:9200/crabtw-%{+YYYY.MM.dd}/_doc" #this should be used to send data to local instance of ES content_type => "application/json; charset=UTF-8" format => "message" - message => '{"producer": "crab", "type": "crabtaskworker", "timestamp":"%{timestamp}", "producer_time":"%{producer_time}", "log_type":"%{log_type}", "slaveID": "%{slaveID}", "workType": "%{workType}", "taskName": "%{taskName}", "completionTime": "%{completionTime}"}' + message => '{"hostname": "%{hostname}", "rec_timestamp":"%{rec_timestamp}", "log_file": "%{log_file}", "producer": "crab", "type": "crabtaskworker", "timestamp":"%{timestamp}", "producer_time":"%{producer_time}", "log_type":"%{log_type}", "slaveID": "%{slaveID}", "workType": "%{workType}", "taskName": "%{taskName}", "completionTime": "%{completionTime}"}' } } @@ -63,7 +171,75 @@ output { #url => "http://localhost:9200/crabtw-%{+YYYY.MM.dd}/_doc" content_type => "application/json; charset=UTF-8" format => "message" - message => '{"producer": "crab", "type": "crabtaskworker", "timestamp":"%{timestamp}", "producer_time":"%{producer_time}", "log_type":"%{log_type}", "slaveID": "%{slaveID}", "functionName": "%{functionName}", "taskID": "%{taskID}", "taskName": "%{taskName}"}' + message => '{"hostname": "%{hostname}", "rec_timestamp":"%{rec_timestamp}", "log_file": "%{log_file}", "producer": "crab", "type": "crabtaskworker", "timestamp":"%{timestamp}", "producer_time":"%{producer_time}", "log_type":"%{log_type}", "slaveID": "%{slaveID}", "functionName": "%{functionName}", "taskID": "%{taskID}", "taskName": "%{taskName}"}' + } + } + + if [log_type] == "successful_publication" { + http { + http_method => post + url => "http://monit-logs.cern.ch:10012/" + content_type => "application/json; charset=UTF-8" + format => "message" + message => '{"hostname": "%{hostname}", "rec_timestamp":"%{rec_timestamp}", "log_file": "%{log_file}", "producer": "crab", "type": "publisher", "timestamp":"%{timestamp}", "producer_time":"%{producer_time}", "log_type":"%{log_type}", "logMsg":"%{logMsg}", "taskName":"%{taskName}", "publicationResult":"%{publicationResult}", "filesPublished":"%{filesPublished}", "blocks":"%{blocks}"}' + } + } + + if [log_type] == "failed_publication" { + http { + http_method => post + url => "http://monit-logs.cern.ch:10012/" + content_type => "application/json; charset=UTF-8" + format => "message" + message => '{"hostname": "%{hostname}", "rec_timestamp":"%{rec_timestamp}", "log_file": "%{log_file}", "producer": "crab", "type": "publisher", "timestamp":"%{timestamp}", "producer_time":"%{producer_time}", "log_type":"%{log_type}", "logMsg":"%{logMsg}", "taskName":"%{taskName}", "blocks":"%{blocks}", "files":"%{files}" }' + } + } + + +####### + if [log_type] == "publisher_config_data" { + http { + http_method => post + url => "http://monit-logs.cern.ch:10012/" + content_type => "application/json; charset=UTF-8" + format => "message" + message => '{"hostname": "%{hostname}", "rec_timestamp":"%{rec_timestamp}", "log_file": "%{log_file}", "producer": "crab", "type": "publisher", "timestamp":"%{timestamp}", "producer_time":"%{producer_time}", "log_type":"%{log_type}", "logMsg":"%{logMsg}", "max_slaves":"%{max_slaves}", "dryRun":"%{dryRun}", "asoworker":"%{asoworker}", "DBShost":"%{DBShost}", "instance":"%{instance}", "version":"%{version}"}' + } + } +####### + + +####### + if [log_type] == "tw_config_data" { + http { + http_method => post + url => "http://monit-logs.cern.ch:10012/" + content_type => "application/json; charset=UTF-8" + format => "message" + message => '{"hostname": "%{hostname}", "rec_timestamp":"%{rec_timestamp}", "log_file": "%{log_file}", "producer": "crab", "type": "crabtaskworker", "timestamp":"%{timestamp}", "producer_time":"%{producer_time}", "log_type":"%{log_type}", "logMsg":"%{logMsg}", "restHost":"%{restHost}", "name":"%{name}", "DBSHostName":"%{DBSHostName}", "instance":"%{instance}", "version":"%{version}", "dbInstance":"%{dbInstance}", "nslaves":"%{nslaves}"}' + } + } +####### + + + if [log_type] == "publication_error" { + http { + http_method => post + url => "http://monit-logs.cern.ch:10012/" + content_type => "application/json; charset=UTF-8" + format => "message" + message => '{"hostname": "%{hostname}", "rec_timestamp":"%{rec_timestamp}", "log_file": "%{log_file}", "producer": "crab", "type": "publisher", "timestamp":"%{timestamp}", "producer_time":"%{producer_time}", "log_type":"%{log_type}", "logMsg":"%{logMsg}", "exceptionHandled":"%{exceptionHandled}" }' + } + } + + + if [log_type] == "acquired_files" { + http { + http_method => post + url => "http://monit-logs.cern.ch:10012/" + content_type => "application/json; charset=UTF-8" + format => "message" + message => '{"hostname": "%{hostname}", "rec_timestamp":"%{rec_timestamp}", "log_file": "%{log_file}", "producer": "crab", "type": "publisher", "timestamp":"%{timestamp}", "producer_time":"%{producer_time}", "log_type":"%{log_type}", "logMsg":"%{logMsg}", "acquiredFiles":"%{acquiredFiles}", "taskName":"%{taskName}" }' } } @@ -75,8 +251,9 @@ output { #url => "http://localhost:9200/crabtw-%{+YYYY.MM.dd}/_doc" content_type => "application/json; charset=UTF-8" format => "message" - message => '{"producer": "crab", "type": "crabtaskworker", "timestamp":"%{timestamp}", "producer_time":"%{producer_time}", "log_type":"%{log_type}", "action": "%{action}", "taskID": "%{taskID}","completionTime": "%{completionTime}", "taskName": "%{taskName}"}' + message => '{"hostname": "%{hostname}", "rec_timestamp":"%{rec_timestamp}", "log_file": "%{log_file}", "producer": "crab", "type": "crabtaskworker", "timestamp":"%{timestamp}", "producer_time":"%{producer_time}", "log_type":"%{log_type}", "action": "%{action}", "taskID": "%{taskID}","completionTime": "%{completionTime}", "taskName": "%{taskName}"}' } } } +