Add t0 function to modify agent code file (#4964)

* Add t0 function to modify agent code file

* Add help message

* Add help message for t0 command

* Add help message for t0 command

* Adding tweak config files to allow t0 --clear-deployment

* t0 command finalized

---------

Co-authored-by: Toine <Antonio Linares>

bin/00_pypi_deploy_prod.sh

@@ -97,6 +97,7 @@ echo "manage=manage"
 sleep 1
 ### The WMCoreVenvVars is a function in the $DEPLOY_DIR/bin/activate file
 declare -A WMCoreVenvVars
+WMCoreVenvVars[TEAM]=$TEAMNAME
 WMCoreVenvVars[WMCORE_CACHE_DIR]=/tmp/$(whoami)
 WMCoreVenvVars[install]=$CURRENT_DIR/install
 WMCoreVenvVars[config]=$CURRENT_DIR/config

bin/00_pypi_deploy_replay.sh

@@ -96,6 +96,7 @@ echo "manage=manage"
 sleep 1
 ### The WMCoreVenvVars is a function in the $DEPLOY_DIR/bin/activate file
 declare -A WMCoreVenvVars
+WMCoreVenvVars[TEAM]=$TEAMNAME
 WMCoreVenvVars[WMCORE_CACHE_DIR]=/tmp/$(whoami)
 WMCoreVenvVars[install]=$CURRENT_DIR/install
 WMCoreVenvVars[config]=$CURRENT_DIR/config

bin/00_pypi_tweak_prod_config.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+CONFIGURATION_FILE=/data/tier0/admin/ProdOfflineConfiguration.py
+sed -i 's+TIER0_CONFIG_FILE+'"$CONFIGURATION_FILE"'+' "$config/config.py"
+sed -i "s+config.Agent.teamName = 'REPLACE_TEAM_NAME'+config.Agent.teamName = '"$TEAMNAME"'+" "$config/config.py"
+sed -i "s+config.Agent.contact = '[email protected]'+config.Agent.contact = '[email protected]'+" "$config/config.py"
+sed -i "s+'team1,team2,cmsdataops'+'tier0production'+g" "$config/config.py"
+sed -i "s+config.RucioInjector.containerDiskRuleParams.*+config.RucioInjector.containerDiskRuleParams = {}+" "$config/config.py"
+echo "config.RucioInjector.blockRuleParams = {}" >> $config/config.py
+sed -i "s+config.RucioInjector.metaDIDProject.*+config.RucioInjector.metaDIDProject = 'Tier0'+" "$config/config.py"
+echo "config.RucioInjector.blockDeletionDelayHours = 168" >> $config/config.py
+##### NOT IN REPLAY #####
+echo "config.RucioInjector.blockDeletionDelayHours = 168" >> $config/config.py
+echo 'config.BossAir.pluginNames = ["SimpleCondorPlugin"]' >> $config/config.py
+echo 'config.JobAccountant.maxAllowedRepackOutputSize = 24 * 1024 * 1024 * 1024' >> $config/config.py
+echo "config.AgentStatusWatcher.runningExpressPercent = 25" >> $config/config.py
+echo "config.AgentStatusWatcher.runningRepackPercent = 10" >> $config/config.py
+echo 'config.TaskArchiver.archiveDelayHours = 2190' >> $config.py
+##### NOT IN REPLAY #####
+echo "config.DBS3Upload.datasetType = 'VALID'" >> $config/config.py
+echo "config.Tier0Feeder.serviceProxy = '$WMA_DEPLOY_DIR/certs/myproxy.pem'" >> $config/config.py
+echo 'config.TaskArchiver.dashBoardUrl = "http://dashb-luminosity.cern.ch/dashboard/request.py/putluminositydata"' >> $config/config.py
+sed -i "s+config.DBS3Upload.uploaderName = 'WMAgent'+config.DBS3Upload.uploaderName = 'T0Prod'+g" "$config/config.py"
+sed -i "s/config.ErrorHandler.maxFailTime.*/config.ErrorHandler.maxFailTime=601200/g" "$config/config.py"
+
+# Enable AgentStatusWatcher - site status automatic updated
+sed -i "s+config.AgentStatusWatcher.ignoreDisks.*+config.AgentStatusWatcher.ignoreDisks = [ '/cvmfs/cvmfs-config.cern.ch', '/cvmfs/cms.cern.ch', '/eos/cms', '/cvmfs/cms-ib.cern.ch', '/cvmfs/patatrack.cern.ch' ]+" "$config/config.py"
+
+
+# password for dropbox upload
+DROPBOX_USER=`cat $WMA_SECRETS_FILE | grep DROPBOX_USER | sed s/DROPBOX_USER=//`
+DROPBOX_PASS=`cat $WMA_SECRETS_FILE | grep DROPBOX_PASS | sed s/DROPBOX_PASS=//`
+
+if [ "x$DROPBOX_USER" == "x" ] || [ "x$DROPBOX_PASS" == "x" ]; then
+    echo "Secrets file doesn't contain DROPBOX_USER or DROPBOX_PASS";
+    exit 1
+fi
+
+echo 'config.Tier0Feeder.dropboxuser = "'$DROPBOX_USER'"' >> $config/config.py
+echo 'config.Tier0Feeder.dropboxpass = "'$DROPBOX_PASS'"' >> $config/config.py

bin/00_pypi_tweak_replay_config.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+CONFIGURATION_FILE=/data/tier0/admin/ReplayOfflineConfiguration.py
+sed -i 's+TIER0_CONFIG_FILE+'"$CONFIGURATION_FILE"'+' "$config/config.py"
+sed -i "s+config.Agent.teamName = 'REPLACE_TEAM_NAME'+config.Agent.teamName = '"$TEAMNAME"'+" "$config/config.py"
+sed -i "s+config.Agent.contact = '[email protected]'+config.Agent.contact = '[email protected]'+" "$config/config.py"
+sed -i "s+'team1,team2,cmsdataops'+'tier0replay'+g" "$config/config.py"
+sed -i "s+config.RucioInjector.containerDiskRuleParams.*+config.RucioInjector.containerDiskRuleParams = {'lifetime': 7 * 24 * 60 * 60}+" "$config/config.py"
+echo "config.RucioInjector.blockRuleParams = {'lifetime': 7 * 24 * 60 * 60}" >> "$config/config.py"
+sed -i "s+config.RucioInjector.metaDIDProject.*+config.RucioInjector.metaDIDProject = 'Test'+" "$config/config.py"
+echo "config.RucioInjector.blockDeletionDelayHours = 2" >> $config/config.py
+echo "config.DBS3Upload.datasetType = 'VALID'" >> $config/config.py
+echo "config.Tier0Feeder.serviceProxy = '$WMA_DEPLOY_DIR/certs/myproxy.pem'" >> $config/config.py
+echo 'config.TaskArchiver.dashBoardUrl = "http://dashb-luminosity.cern.ch/dashboard/request.py/putluminositydata"' >> $config/config.py
+sed -i "s+config.DBS3Upload.uploaderName = 'WMAgent'+config.DBS3Upload.uploaderName = 'T0Replay'+g" "$config/config.py"
+#Overwrite ErrorHandler to avoid jobs going to ensure T0 jobs can always be retried
+sed -i "s/config.ErrorHandler.maxFailTime.*/config.ErrorHandler.maxFailTime=601200/g" "$config/config.py"
+sed -i "s+config.RucioInjector.listTiersToInject.*+config.RucioInjector.listTiersToInject = ['AOD', 'MINIAOD', 'NANOAOD', 'NANOAODSIM', 'RAW', 'FEVT', 'USER', 'ALCARECO', 'ALCAPROMPT','DQMIO','RAW-RECO']+" "$config/config.py"
+
+##### ONLY IN REPLAY #####
+sed -i "s/config.RetryManager.plugins.*/config.RetryManager.plugins={'default': 'PauseAlgo', 'Cleanup': 'PauseAlgo', 'LogCollect': 'PauseAlgo'}/g" "$config/config.py"
+sed -i "s/config.ErrorHandler.maxRetries.*/config.ErrorHandler.maxRetries={'default': 30, 'Cleanup': 30, 'LogCollect': 30}/g" "$config/config.py"
+##### ONLY IN REPLAY #####
+
+# Enable AgentStatusWatcher - site status automatic updated
+sed -i "s+config.AgentStatusWatcher.ignoreDisks.*+config.AgentStatusWatcher.ignoreDisks = [ '/cvmfs/cvmfs-config.cern.ch', '/cvmfs/cms.cern.ch', '/eos/cms', '/cvmfs/cms-ib.cern.ch', '/cvmfs/patatrack.cern.ch' ]+" "$config/config.py"
+
+# password for dropbox upload
+
+DROPBOX_USER=`cat $WMA_SECRETS_FILE | grep DROPBOX_USER | sed s/DROPBOX_USER=//`
+DROPBOX_PASS=`cat $WMA_SECRETS_FILE | grep DROPBOX_PASS | sed s/DROPBOX_PASS=//`
+
+if [ "x$DROPBOX_USER" == "x" ] || [ "x$DROPBOX_PASS" == "x" ]; then
+    echo "Secrets file doesn't contain DROPBOX_USER or DROPBOX_PASS";
+    exit 1
+fi
+
+echo 'config.Tier0Feeder.dropboxuser = "'$DROPBOX_USER'"' >> $config/config.py
+echo 'config.Tier0Feeder.dropboxpass = "'$DROPBOX_PASS'"' >> $config/config.py
+
+##### ONLY IN PROD #####
+#echo "config.RucioInjector.blockDeletionDelayHours = 168" >> ./config/tier0/config.py
+#echo 'config.BossAir.pluginNames = ["SimpleCondorPlugin"]' >> ./config/tier0/config.py
+#echo 'config.JobAccountant.maxAllowedRepackOutputSize = 24 * 1024 * 1024 * 1024' >> ./config/tier0/config.py
+#echo "config.AgentStatusWatcher.runningExpressPercent = 25" >> ./config/tier0/config.py
+#echo "config.AgentStatusWatcher.runningRepackPercent = 10" >> ./config/tier0/config.py
+#echo 'config.TaskArchiver.archiveDelayHours = 2190' >> $config.py
+##### ONLY IN PROD #####

bin/pypi_update.sh

@@ -16,6 +16,8 @@ declare -A PYPI_DEPLOYMENT_SCRIPTS=(
     [00_pypi_patches.sh]=https://raw.githubusercontent.com/dmwm/T0/master/bin/00_pypi_patches.sh
     [00_pypi_deploy_replay.sh]=https://raw.githubusercontent.com/dmwm/T0/master/bin/00_pypi_deploy_replay.sh
     [00_pypi_deploy_prod.sh]=https://raw.githubusercontent.com/dmwm/T0/master/bin/00_pypi_deploy_prod.sh
+    [00_pypi_tweak_replay_config.sh]=https://raw.githubusercontent.com/dmwm/T0/master/bin/00_pypi_tweak_replay_config.sh
+    [00_pypi_tweak_prod_config.sh]=https://raw.githubusercontent.com/dmwm/T0/master/bin/00_pypi_tweak_prod_config.sh
 )
 
 for SCRIPT in "${!PYPI_DEPLOYMENT_SCRIPTS[@]}"; do

bin/t0

@@ -6,10 +6,19 @@ import traceback
 import os
 
 help_message = '''
-The help message goes here.
+- t0 --update-t0=<version>          : Uninstalls current T0 to install sepcified T0 version
+-
+- t0 --start-agent                  : Starts the agent
+-
+- t0 --stop-agent                   : Stops the agent
+-
+- t0 --clear-deployment             : Start a new deployment without re-installing the software
+-
+- t0 --resource-control=<site>      : Adds input site to resource control only for processing jobs
+-
+- t0 --edit-code=<file>             : Looks for input file within site-packages directory and opens it for edit using vim
 '''
 
-
 class Usage(Exception):
     def __init__(self, msg):
         self.msg = msg
@@ -31,20 +40,30 @@ def stop ():
     print("Stopping the agent")
     os.system('manage stop-agent')
 
-def updateT0 ():
+def updateT0 (version):
     """
     _updateT0_
     Updates the T0 version
     """
-    print("Updating T0")
+    deployDir = os.getenv('WMA_DEPLOY_DIR')
+    os.system('echo "Stopping the agent"')
+    os.system('sleep 3')
+    stop()
+    os.system('echo "Now updating T0"')
+    os.system('sleep 2')
     os.system('pip uninstall -y T0')
-    os.system('pip install T0')
+    os.system('pip install T0=={}'.format(version))
+    os.system('chmod +x {}/bin/t0'.format(deployDir))
+    os.system('chmod +x {}/etc/Tier0Config.py'.format(deployDir))
+    start()
+    os.system('echo "Done"')
 
 def updateWMCore ():
     """
     _updateWMCore_
     Updates the T0 version
     """
+    return
     print("Updating WMCore")
     os.system('pip uninstall -y wmagent')
     os.system('pip install wmagent')
@@ -66,21 +85,38 @@ def wipeT0ast (userInput='n'):
     else:
         os.system('echo "Not wiping T0AST"')
 
+def tweakConfig (teamname):
+    """
+    _tweakConfig_
+    Modifies config.py file according to type of agent: replay or prod
+    """
+    if teamname == 'Tier0Replay':
+        os.system('bash /data/tier0/00_pypi_tweak_replay_config.sh')
+    elif teamname == 'Tier0Production':
+        os.system('bash /data/tier0/00_pypi_tweak_prod_config.sh')
+    else:
+        os.system('echo "Something is not right with the TEAM provided: {}"'.format(teamname))
+
 def clearDeployment (userInput='n'):
     """
     _clearDeployment_
     Avoid re-deploying from scratch when unnecessary
     """
-    if userInput == 'Y':
-        resetCouch()
-        wipeT0ast(userInput=userInput)
-        os.system('rm -rf $WMA_CONFIG_DIR/.init*')
-        os.system('rm -rf $WMA_INSTALL_DIR/*')
-        os.system('rm -rf /data/tier0/admin/Specs/*')
-        init()
-    else:
+    if userInput != 'Y':
         os.system('echo "Not performing anything"')
-        os.system('sleep 2')
+        return
+    teamname = os.getenv('TEAM')
+    print(teamname)
+    stop()
+    os.system('condor_rm -all')
+    resetCouch()
+    wipeT0ast(userInput=userInput)
+    os.system('rm -rf $WMA_CONFIG_DIR/.init*')
+    os.system('rm -rf $WMA_INSTALL_DIR/*')
+    os.system('rm -rf /data/tier0/admin/Specs/*')
+    init()
+    tweakConfig(teamname)
+    os.system('echo "Done. Agent is ready to start"')
 
 def init ():
     """
@@ -96,14 +132,35 @@ def addProcessingSite(site=None):
     """
     command_1 = "manage execute-agent wmagent-resource-control --site-name={} --cms-name={} --pnn={} --ce-name={} --pending-slots=20000 --running-slots=20000 --plugin=SimpleCondorPlugin".format(site, site, site, site)
     command_2 = "manage execute-agent wmagent-resource-control --site-name={} --task-type=Processing --pending-slots=10000 --running-slots=10000".format(site)
+    os.system(command_1)
+    os.system(command_2)
+
+def modifyCodeFile(file=None):
+    """
+    Simplifies the modification of the agent code
+    Goes to the site-packages directory and looks for file to modify, if given
+    """
+    deployDir = os.getenv('WMA_DEPLOY_DIR')
+    packagePath = os.path.join(deployDir, 'lib/python3.9/site-packages')
+    os.chdir(packagePath)
+
+    findFile = 'find . -name {}'.format(file)
+    with os.popen(findFile) as stream:
+        filePath = stream.read().strip()
+    filePath = os.path.join(packagePath, filePath[2:])
+    os.system('echo "Now modifying {}"'.format(filePath))
+    os.system('sleep 3')
+    os.system('vim {}'.format(filePath))
+    os.system('echo "Done"')
+
 def main(argv=None):
     if argv is None:
         argv = sys.argv
 
     try:
         try:
             opts, args = getopt.getopt(argv[1:], "h",
-            ["help", "update-t0", "update-wmcore", "start-agent", "stop-agent", "clear-deployment", "resource-control" ])
+            ["help", "update-t0=", "start-agent", "stop-agent", "clear-deployment", "resource-control=", "edit-code="])
 
         except getopt.error as msg:
             raise Usage(msg)
@@ -114,10 +171,7 @@ def main(argv=None):
                 raise Usage(help_message)
 
             if option == "--update-t0":
-                updateT0()
-
-            if option == "--update-wmcore":
-                updateWMCore()
+                updateT0(value)
 
             if option == "--start-agent":
                 start()
@@ -140,6 +194,13 @@ def main(argv=None):
                 site = value
                 addProcessingSite(site)
 
+            if option == "--edit-code":
+                if len(value) > 0:
+                    modifyCodeFile(value)
+                else:
+                    os.system('echo "Please provide a file to modify"')
+                    os.system('echo "t0 --edit-code={file}"')
+
     except Usage as err:
         print(sys.argv[0].split("/")[-1] + ": " + str(err.msg), file=sys.stderr)
         print("\t for help use --help", file=sys.stderr)