Skip to content

Commit

Permalink
Port to python3 recent small fixes from master (#6858)
Browse files Browse the repository at this point in the history 
* use gocurl from CVMFS Fix #6822 (#6823)

* add comment about py2/3 compatibility needs (#6826)

* add GH remote for Diego

* upload new config version (#6852)

* stop trying to remove failed migrations from 2019. Fix #6854 (#6855)

Co-authored-by: Daina <[email protected]>
  • Loading branch information
@belforte @ddaina
belforte and ddaina authored Nov 11, 2021
1 parent 9d42f7b commit 1ed4d71
Show file tree
Hide file tree
Showing 2 changed files with 182 additions and 3 deletions.
2 changes: 2 additions & 0 deletions Docker/addGH.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,14 @@ git checkout $CRABServerTag
git remote add stefano https://github.com/belforte/CRABServer.git
git remote add daina https://github.com/ddaina/CRABServer.git
git remote add mapellidario https://github.com/mapellidario/CRABServer.git
git remote add diego https://github.com/dciangot/CRABServer.git
cd ..
cd WMCore
git checkout $WMCoreTag
git remote add stefano https://github.com/belforte/WMCore.git
git remote add daina https://github.com/ddaina/WMCore.git
git remote add mapellidario https://github.com/mapellidario/WMCore.git
git remote add diego https://github.com/dciangot/WMCore.git

# 4. create .gitconfig
cat > ~/.gitconfig << EOF
Expand Down
183 changes: 180 additions & 3 deletions src/script/Monitor/logstash/crabtaskworker.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,13 @@ input {
}

filter{

mutate { add_field => {
"hostname" => "%{[cloud][instance][name]}"
"log_file" => "%{[log][file][path]}"
}
}

if [message] =~ /.*completed in.*/ {
grok{
match => {
Expand All @@ -14,6 +21,34 @@ filter{
add_field => {"log_type" => "work_on_task_completed"}
overwrite => ["message"]
}


###BELOW
} else if [message] =~ /.*PUBSTART.*/ {
grok{
match => {
#2021-05-19 09:15:34,006:INFO:PublisherMaster,149:PUBSTART: {"max_slaves": 5, "dryRun": false, "asoworker": "schedd", "DBShost": "cmsweb-prod.cern.ch", "instance": "test2", "version": "v3.210514"}
"message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:PUBSTART:%{GREEDYDATA:publisher_json_data}"
}
add_field => {"log_type" => "publisher_config_data"}
overwrite => ["message"]
}
###END


###BELOW
} else if [message] =~ /.*TWSTART.*/ {
grok{
match => {
#2021-05-25 18:50:55,460:INFO:MasterWorker,174:TWSTART: {"restHost": "X", "name": "Y", "recurringActions": ["RemovetmpDir", "BanDestinationSites", "TapeRecallStatus"], "DBSHostName": "Z", "instance": "other", "version": "development", "dbInstance": "dev", "nslaves": 1}
"message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:MasterWorker,%{INT}:TWSTART: %{GREEDYDATA:tw_json_data}"
}
add_field => {"log_type" => "tw_config_data"}
overwrite => ["message"]
}
###END


} else if [message] =~ /.*Starting.*at.*on.*/ {
grok{
match => {
Expand All @@ -23,6 +58,47 @@ filter{
add_field => {"log_type" => "start_new_task"}
overwrite => ["message"]
}

} else if [message] =~ /.*blocks failed.*files.*/ {
grok{
match => {
#2020-11-21 01:45:32,789:ERROR:PublisherMaster,550:Taskname 201118_182833:vcepaiti_crab_QCD_Pt-80to120_EMEnriched_TuneCUETP8M1_13TeV_pythia8-2016_NANOX_201117 : 1 blocks failed for a total of 11 files
"message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:Taskname %{NOTSPACE:taskName} : %{INT:blocks} blocks failed for a total of %{INT:files} files"
}
add_field => {"log_type" => "failed_publication"}
overwrite => ["message"]
}

} else if [message] =~ /.*Published.*files in.*/ {
grok{
match => {
#2020-11-27 14:25:16,191:INFO:PublisherMaster,545:Taskname 201127_011713:anstahll_crab_AODSkim_HIMinimumBias14_HIRun2018_04Apr2019_DiMuMassMin2_20201117 is OK. Published 37 files in 1 blocks.
"message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:Taskname %{NOTSPACE:taskName} is %{NOTSPACE:publicationResult}. Published %{INT:filesPublished} files in %{INT:blocks} blocks."
}
add_field => {"log_type" => "successful_publication"}
overwrite => ["message"]
}

} else if [message] =~ /.*Exception.*TaskPublish.*/ {
grok{
match => {
#2020-11-26 19:23:27,737:ERROR:PublisherMaster,554:Exception when calling TaskPublish!
"message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:%{GREEDYDATA:exceptionHandled}"
}
add_field => {"log_type" => "publication_error"}
overwrite => ["message"]
}

} else if [message] =~ /.*DEBUG:master.*/ {
grok{
match => {
#2021-04-15 22:23:39,566:DEBUG:master: 8 : 210415_093249:algomez_crab_QCDHT100to200TuneCP5PSWeights13TeV-madgraphMLM
"message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:master:%{SPACE}%{INT:acquiredFiles}%{SPACE}:%{SPACE}%{NOTSPACE:taskName}"
}
add_field => {"log_type" => "acquired_files"}
overwrite => ["message"]
}

} else if [message] =~ /.*Finished.*object at .*/ {
grok{
match => {
Expand All @@ -40,10 +116,42 @@ filter{
}
ruby { code => "event.set('timestamp', (event.get('timestamp_str').to_f * 1000).to_i)" }
ruby { code => "event.set('producer_time', (event.get('@timestamp').to_f * 1000).to_i)" }
ruby { code => "event.set('rec_timestamp', (event.get('timestamp_str').to_f * 1000).to_i)" }

if [publisher_json_data] {
json{
source => "publisher_json_data"
target => "publisher_json_data"
}

ruby {
code => '
event.get("publisher_json_data").each { |k, v|
event.set(k,v)
}
event.remove("publisher_json_data")
'
}
} else if [tw_json_data] {
json{
source => "tw_json_data"
target => "tw_json_data"
}

ruby {
code => '
event.get("tw_json_data").each { |k, v|
event.set(k,v)
}
event.remove("tw_json_data")
'
}
}
}




output {
if [log_type] == "work_on_task_completed" {
http {
Expand All @@ -52,7 +160,7 @@ output {
#url => "http://localhost:9200/crabtw-%{+YYYY.MM.dd}/_doc" #this should be used to send data to local instance of ES
content_type => "application/json; charset=UTF-8"
format => "message"
message => '{"producer": "crab", "type": "crabtaskworker", "timestamp":"%{timestamp}", "producer_time":"%{producer_time}", "log_type":"%{log_type}", "slaveID": "%{slaveID}", "workType": "%{workType}", "taskName": "%{taskName}", "completionTime": "%{completionTime}"}'
message => '{"hostname": "%{hostname}", "rec_timestamp":"%{rec_timestamp}", "log_file": "%{log_file}", "producer": "crab", "type": "crabtaskworker", "timestamp":"%{timestamp}", "producer_time":"%{producer_time}", "log_type":"%{log_type}", "slaveID": "%{slaveID}", "workType": "%{workType}", "taskName": "%{taskName}", "completionTime": "%{completionTime}"}'
}
}

Expand All @@ -63,7 +171,75 @@ output {
#url => "http://localhost:9200/crabtw-%{+YYYY.MM.dd}/_doc"
content_type => "application/json; charset=UTF-8"
format => "message"
message => '{"producer": "crab", "type": "crabtaskworker", "timestamp":"%{timestamp}", "producer_time":"%{producer_time}", "log_type":"%{log_type}", "slaveID": "%{slaveID}", "functionName": "%{functionName}", "taskID": "%{taskID}", "taskName": "%{taskName}"}'
message => '{"hostname": "%{hostname}", "rec_timestamp":"%{rec_timestamp}", "log_file": "%{log_file}", "producer": "crab", "type": "crabtaskworker", "timestamp":"%{timestamp}", "producer_time":"%{producer_time}", "log_type":"%{log_type}", "slaveID": "%{slaveID}", "functionName": "%{functionName}", "taskID": "%{taskID}", "taskName": "%{taskName}"}'
}
}

if [log_type] == "successful_publication" {
http {
http_method => post
url => "http://monit-logs.cern.ch:10012/"
content_type => "application/json; charset=UTF-8"
format => "message"
message => '{"hostname": "%{hostname}", "rec_timestamp":"%{rec_timestamp}", "log_file": "%{log_file}", "producer": "crab", "type": "publisher", "timestamp":"%{timestamp}", "producer_time":"%{producer_time}", "log_type":"%{log_type}", "logMsg":"%{logMsg}", "taskName":"%{taskName}", "publicationResult":"%{publicationResult}", "filesPublished":"%{filesPublished}", "blocks":"%{blocks}"}'
}
}

if [log_type] == "failed_publication" {
http {
http_method => post
url => "http://monit-logs.cern.ch:10012/"
content_type => "application/json; charset=UTF-8"
format => "message"
message => '{"hostname": "%{hostname}", "rec_timestamp":"%{rec_timestamp}", "log_file": "%{log_file}", "producer": "crab", "type": "publisher", "timestamp":"%{timestamp}", "producer_time":"%{producer_time}", "log_type":"%{log_type}", "logMsg":"%{logMsg}", "taskName":"%{taskName}", "blocks":"%{blocks}", "files":"%{files}" }'
}
}


#######
if [log_type] == "publisher_config_data" {
http {
http_method => post
url => "http://monit-logs.cern.ch:10012/"
content_type => "application/json; charset=UTF-8"
format => "message"
message => '{"hostname": "%{hostname}", "rec_timestamp":"%{rec_timestamp}", "log_file": "%{log_file}", "producer": "crab", "type": "publisher", "timestamp":"%{timestamp}", "producer_time":"%{producer_time}", "log_type":"%{log_type}", "logMsg":"%{logMsg}", "max_slaves":"%{max_slaves}", "dryRun":"%{dryRun}", "asoworker":"%{asoworker}", "DBShost":"%{DBShost}", "instance":"%{instance}", "version":"%{version}"}'
}
}
#######


#######
if [log_type] == "tw_config_data" {
http {
http_method => post
url => "http://monit-logs.cern.ch:10012/"
content_type => "application/json; charset=UTF-8"
format => "message"
message => '{"hostname": "%{hostname}", "rec_timestamp":"%{rec_timestamp}", "log_file": "%{log_file}", "producer": "crab", "type": "crabtaskworker", "timestamp":"%{timestamp}", "producer_time":"%{producer_time}", "log_type":"%{log_type}", "logMsg":"%{logMsg}", "restHost":"%{restHost}", "name":"%{name}", "DBSHostName":"%{DBSHostName}", "instance":"%{instance}", "version":"%{version}", "dbInstance":"%{dbInstance}", "nslaves":"%{nslaves}"}'
}
}
#######


if [log_type] == "publication_error" {
http {
http_method => post
url => "http://monit-logs.cern.ch:10012/"
content_type => "application/json; charset=UTF-8"
format => "message"
message => '{"hostname": "%{hostname}", "rec_timestamp":"%{rec_timestamp}", "log_file": "%{log_file}", "producer": "crab", "type": "publisher", "timestamp":"%{timestamp}", "producer_time":"%{producer_time}", "log_type":"%{log_type}", "logMsg":"%{logMsg}", "exceptionHandled":"%{exceptionHandled}" }'
}
}


if [log_type] == "acquired_files" {
http {
http_method => post
url => "http://monit-logs.cern.ch:10012/"
content_type => "application/json; charset=UTF-8"
format => "message"
message => '{"hostname": "%{hostname}", "rec_timestamp":"%{rec_timestamp}", "log_file": "%{log_file}", "producer": "crab", "type": "publisher", "timestamp":"%{timestamp}", "producer_time":"%{producer_time}", "log_type":"%{log_type}", "logMsg":"%{logMsg}", "acquiredFiles":"%{acquiredFiles}", "taskName":"%{taskName}" }'
}
}

Expand All @@ -75,8 +251,9 @@ output {
#url => "http://localhost:9200/crabtw-%{+YYYY.MM.dd}/_doc"
content_type => "application/json; charset=UTF-8"
format => "message"
message => '{"producer": "crab", "type": "crabtaskworker", "timestamp":"%{timestamp}", "producer_time":"%{producer_time}", "log_type":"%{log_type}", "action": "%{action}", "taskID": "%{taskID}","completionTime": "%{completionTime}", "taskName": "%{taskName}"}'
message => '{"hostname": "%{hostname}", "rec_timestamp":"%{rec_timestamp}", "log_file": "%{log_file}", "producer": "crab", "type": "crabtaskworker", "timestamp":"%{timestamp}", "producer_time":"%{producer_time}", "log_type":"%{log_type}", "action": "%{action}", "taskID": "%{taskID}","completionTime": "%{completionTime}", "taskName": "%{taskName}"}'
}
}

}

0 comments on commit 1ed4d71

Please sign in to comment.