-
Notifications
You must be signed in to change notification settings - Fork 786
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
bypass runcmsgrid patch for CI and update requests in CI
- Loading branch information
Showing
5 changed files
with
204 additions
and
21 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| import FWCore.ParameterSet.Config as cms | ||
|
|
||
| externalLHEProducer = cms.EDProducer("ExternalLHEProducer", | ||
| args = cms.vstring('/cvmfs/cms.cern.ch/phys_generator/gridpacks/slc7_amd64_gcc700/13TeV/madgraph/V5_2.6.5/ttbar_LFV_SMEFT_splitFlavour/SMEFTsim_TT_vector_emutu_slc7_amd64_gcc700_CMSSW_10_6_19_tarball.tar.xz'), | ||
| nEvents = cms.untracked.uint32(5000), | ||
| numberOfParameters = cms.uint32(1), | ||
| outputFile = cms.string('cmsgrid_final.lhe'), | ||
| generateConcurrently = cms.untracked.bool(False), | ||
| scriptName = cms.FileInPath('GeneratorInterface/LHEInterface/data/run_generic_tarball_cvmfs.sh') | ||
| ) | ||
|
|
||
|
|
||
| from Configuration.Generator.Pythia8CommonSettings_cfi import * | ||
| from Configuration.Generator.MCTunes2017.PythiaCP5Settings_cfi import * | ||
| from Configuration.Generator.PSweightsPythia.PythiaPSweightsSettings_cfi import * | ||
|
|
||
| generator = cms.EDFilter("Pythia8ConcurrentHadronizerFilter", | ||
| maxEventsToPrint = cms.untracked.int32(1), | ||
| pythiaPylistVerbosity = cms.untracked.int32(1), | ||
| filterEfficiency = cms.untracked.double(1.0), | ||
| pythiaHepMCVerbosity = cms.untracked.bool(False), | ||
| comEnergy = cms.double(13000.), | ||
| PythiaParameters = cms.PSet( | ||
| pythia8CommonSettingsBlock, | ||
| pythia8CP5SettingsBlock, | ||
| pythia8PSweightsSettingsBlock, | ||
| parameterSets = cms.vstring('pythia8CommonSettings', | ||
| 'pythia8CP5Settings', | ||
| 'pythia8PSweightsSettings', | ||
| ) | ||
| ) | ||
| ) | ||
| ProductionFilterSequence = cms.Sequence(generator) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,155 @@ | ||
| #!/bin/bash | ||
|
|
||
| # Binds for singularity containers | ||
| # Mount /afs, /eos, /cvmfs, /etc/grid-security for xrootd | ||
| export APPTAINER_BINDPATH='/afs,/cvmfs,/cvmfs/grid.cern.ch/etc/grid-security:/etc/grid-security,/eos,/etc/pki/ca-trust,/run/user,/var/run/user' | ||
|
|
||
| cat <<'EndOfGenScriptFile' > TOP-RunIISummer20UL16wmLHEGEN-00883_gen_script.sh | ||
| #!/bin/bash | ||
| echo "Running CMS GEN request script using cms-sw containers. Architecture: el9:x86_64" | ||
| python3.9 -m venv cms_gen_venv_TOP-RunIISummer20UL16wmLHEGEN-00883 && source ./cms_gen_venv_TOP-RunIISummer20UL16wmLHEGEN-00883/bin/activate | ||
| # Install the PdmV REST client | ||
| pip install git+https://github.com/cms-PdmV/mcm_scripts &> /dev/null | ||
| echo "Packages installed" | ||
| pip freeze | ||
| echo "" | ||
| # GEN Script begin | ||
| rm -f request_fragment_check.py | ||
| wget -q https://raw.githubusercontent.com/cms-sw/genproductions/master/bin/utils/request_fragment_check.py | ||
| chmod +x request_fragment_check.py | ||
| ./request_fragment_check.py --bypass_status --prepid TOP-RunIISummer20UL16wmLHEGEN-00883 | ||
| # End of CMS GEN script file: TOP-RunIISummer20UL16wmLHEGEN-00883_gen_script.sh | ||
| EndOfGenScriptFile | ||
| chmod +x TOP-RunIISummer20UL16wmLHEGEN-00883_gen_script.sh | ||
|
|
||
| # Run in singularity container | ||
| singularity run --home $PWD:$PWD /cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el9:x86_64 $(echo $(pwd)/TOP-RunIISummer20UL16wmLHEGEN-00883_gen_script.sh) | ||
|
|
||
| GEN_ERR=$? | ||
| if [ $GEN_ERR -ne 0 ]; then | ||
| echo "GEN Checking Script returned exit code $GEN_ERR which means there are $GEN_ERR errors" | ||
| echo "Validation WILL NOT RUN" | ||
| echo "Please correct errors in the request and run validation again" | ||
| exit $GEN_ERR | ||
| fi | ||
| echo "Running VALIDATION. GEN Request Checking Script returned no errors" | ||
| # GEN Script end | ||
|
|
||
| # Download fragment from McM | ||
| curl -s -k https://cms-pdmv-prod.web.cern.ch/mcm/public/restapi/requests/get_fragment/TOP-RunIISummer20UL16wmLHEGEN-00883 --retry 3 --create-dirs -o Configuration/GenProduction/python/TOP-RunIISummer20UL16wmLHEGEN-00883-fragment.py | ||
| [ -s Configuration/GenProduction/python/TOP-RunIISummer20UL16wmLHEGEN-00883-fragment.py ] || exit $?; | ||
|
|
||
| # Check if fragment contais gridpack path ant that it is in cvmfs | ||
| if grep -q "gridpacks" Configuration/GenProduction/python/TOP-RunIISummer20UL16wmLHEGEN-00883-fragment.py; then | ||
| if ! grep -q "/cvmfs/cms.cern.ch/phys_generator/gridpacks" Configuration/GenProduction/python/TOP-RunIISummer20UL16wmLHEGEN-00883-fragment.py; then | ||
| echo "Gridpack inside fragment is not in cvmfs." | ||
| exit -1 | ||
| fi | ||
| fi | ||
|
|
||
| # Dump actual test code to a TOP-RunIISummer20UL16wmLHEGEN-00883_test.sh file that can be run in Singularity | ||
| cat <<'EndOfTestFile' > TOP-RunIISummer20UL16wmLHEGEN-00883_test.sh | ||
| #!/bin/bash | ||
| export SCRAM_ARCH=slc7_amd64_gcc700 | ||
| source /cvmfs/cms.cern.ch/cmsset_default.sh | ||
| if [ -r CMSSW_10_6_43/src ] ; then | ||
| echo release CMSSW_10_6_43 already exists | ||
| else | ||
| scram p CMSSW CMSSW_10_6_43 | ||
| fi | ||
| cd CMSSW_10_6_43/src | ||
| eval `scram runtime -sh` | ||
| mv ../../Configuration . | ||
| scram b | ||
| cd ../.. | ||
| # Maximum validation duration: 57600s | ||
| # Margin for validation duration: 30% | ||
| # Validation duration with margin: 57600 * (1 - 0.30) = 40320s | ||
| # Time per event for each sequence: 2.0000s | ||
| # Threads for each sequence: 1 | ||
| # Time per event for single thread for each sequence: 1 * 2.0000s = 2.0000s | ||
| # Which adds up to 2.0000s per event | ||
| # Single core events that fit in validation duration: 40320s / 2.0000s = 20160 | ||
| # Produced events limit in McM is 10000 | ||
| # According to 1.0000 efficiency, validation should run 10000 / 1.0000 = 10000 events to reach the limit of 10000 | ||
| # Take the minimum of 20160 and 10000, but more than 0 -> 10000 | ||
| # It is estimated that this validation will produce: 10000 * 1.0000 = 10000 events | ||
| EVENTS=10000 | ||
| # cmsDriver command | ||
| cmsDriver.py Configuration/GenProduction/python/TOP-RunIISummer20UL16wmLHEGEN-00883-fragment.py --python_filename TOP-RunIISummer20UL16wmLHEGEN-00883_1_cfg.py --eventcontent RAWSIM,LHE --customise Configuration/DataProcessing/Utils.addMonitoring --datatier GEN,LHE --fileout file:TOP-RunIISummer20UL16wmLHEGEN-00883.root --conditions 106X_mcRun2_asymptotic_v13 --beamspot Realistic25ns13TeV2016Collision --customise_commands process.source.numberEventsInLuminosityBlock="cms.untracked.uint32(100)" --step LHE,GEN --geometry DB:Extended --era Run2_2016 --no_exec --mc -n $EVENTS || exit $? ; | ||
| # Run generated config | ||
| REPORT_NAME=TOP-RunIISummer20UL16wmLHEGEN-00883_report.xml | ||
| # Run the cmsRun | ||
| cmsRun -e -j $REPORT_NAME TOP-RunIISummer20UL16wmLHEGEN-00883_1_cfg.py || exit $? ; | ||
| # Parse values from TOP-RunIISummer20UL16wmLHEGEN-00883_report.xml report | ||
| processedEvents=$(grep -Po "(?<=<Metric Name=\"NumberEvents\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1) | ||
| producedEvents=$(grep -Po "(?<=<TotalEvents>)(\d*)(?=</TotalEvents>)" $REPORT_NAME | tail -n 1) | ||
| threads=$(grep -Po "(?<=<Metric Name=\"NumberOfThreads\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1) | ||
| peakValueRss=$(grep -Po "(?<=<Metric Name=\"PeakValueRss\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1) | ||
| peakValueVsize=$(grep -Po "(?<=<Metric Name=\"PeakValueVsize\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1) | ||
| totalSize=$(grep -Po "(?<=<Metric Name=\"Timing-tstoragefile-write-totalMegabytes\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1) | ||
| totalSizeAlt=$(grep -Po "(?<=<Metric Name=\"Timing-file-write-totalMegabytes\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1) | ||
| totalJobTime=$(grep -Po "(?<=<Metric Name=\"TotalJobTime\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1) | ||
| totalJobCPU=$(grep -Po "(?<=<Metric Name=\"TotalJobCPU\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1) | ||
| eventThroughput=$(grep -Po "(?<=<Metric Name=\"EventThroughput\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1) | ||
| avgEventTime=$(grep -Po "(?<=<Metric Name=\"AvgEventTime\" Value=\")(.*)(?=\"/>)" $REPORT_NAME | tail -n 1) | ||
| if [ -z "$threads" ]; then | ||
| echo "Could not find NumberOfThreads in report, defaulting to 1" | ||
| threads=1 | ||
| fi | ||
| if [ -z "$eventThroughput" ]; then | ||
| eventThroughput=$(bc -l <<< "scale=4; 1 / ($avgEventTime / $threads)") | ||
| fi | ||
| if [ -z "$totalSize" ]; then | ||
| totalSize=$totalSizeAlt | ||
| fi | ||
| if [ -z "$processedEvents" ]; then | ||
| processedEvents=$EVENTS | ||
| fi | ||
| echo "Validation report of TOP-RunIISummer20UL16wmLHEGEN-00883 sequence 1/1" | ||
| echo "Processed events: $processedEvents" | ||
| echo "Produced events: $producedEvents" | ||
| echo "Threads: $threads" | ||
| echo "Peak value RSS: $peakValueRss MB" | ||
| echo "Peak value Vsize: $peakValueVsize MB" | ||
| echo "Total size: $totalSize MB" | ||
| echo "Total job time: $totalJobTime s" | ||
| echo "Total CPU time: $totalJobCPU s" | ||
| echo "Event throughput: $eventThroughput" | ||
| echo "CPU efficiency: "$(bc -l <<< "scale=2; ($totalJobCPU * 100) / ($threads * $totalJobTime)")" %" | ||
| echo "Size per event: "$(bc -l <<< "scale=4; ($totalSize * 1024 / $producedEvents)")" kB" | ||
| echo "Time per event: "$(bc -l <<< "scale=4; (1 / $eventThroughput)")" s" | ||
| echo "Filter efficiency percent: "$(bc -l <<< "scale=8; ($producedEvents * 100) / $processedEvents")" %" | ||
| echo "Filter efficiency fraction: "$(bc -l <<< "scale=10; ($producedEvents) / $processedEvents") | ||
| # End of TOP-RunIISummer20UL16wmLHEGEN-00883_test.sh file | ||
| EndOfTestFile | ||
|
|
||
| # Make file executable | ||
| chmod +x TOP-RunIISummer20UL16wmLHEGEN-00883_test.sh | ||
|
|
||
| if [ -e "/cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el7:amd64" ]; then | ||
| CONTAINER_NAME="el7:amd64" | ||
| elif [ -e "/cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el7:x86_64" ]; then | ||
| CONTAINER_NAME="el7:x86_64" | ||
| else | ||
| echo "Could not find amd64 or x86_64 for el7" | ||
| exit 1 | ||
| fi | ||
| # Run in singularity container | ||
| export SINGULARITY_CACHEDIR="/tmp/$(whoami)/singularity" | ||
| singularity run --home $PWD:$PWD /cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/$CONTAINER_NAME $(echo $(pwd)/TOP-RunIISummer20UL16wmLHEGEN-00883_test.sh) |
Oops, something went wrong.