Merge pull request #7873 from fstagni/80_tests_add_systemTests

[8.0] added some tests jobs for certifications

src/DIRAC/tests/Utilities/testJobDefinitions.py

@@ -3,14 +3,13 @@
 
 # pylint: disable=invalid-name
 
+import errno
 import os
 import time
-import errno
 
 from DIRAC import rootPath
-from DIRAC.Interfaces.API.Job import Job
 from DIRAC.Interfaces.API.Dirac import Dirac
-
+from DIRAC.Interfaces.API.Job import Job
 from DIRAC.tests.Utilities.utils import find_all
 
 
@@ -65,6 +64,44 @@ def helloWorld():
     return endOfAllJobs(J)
 
 
+def helloWorld_input():
+    """simple hello world job with input"""
+
+    J = baseToAllJobs("helloWorld_input")
+    try:
+        J.setInputSandbox([find_all("exe-script-with-input.py", rootPath, "DIRAC/tests/Workflow")[0]])
+    except IndexError:
+        try:
+            J.setInputSandbox([find_all("exe-script-with-input.py", ".", "DIRAC/tests/Workflow")[0]])
+        except IndexError:  # we are in Jenkins
+            J.setInputSandbox(
+                [find_all("exe-script-with-input.py", os.environ["WORKSPACE"], "DIRAC/tests/Workflow")[0]]
+            )
+    J.setExecutable("exe-script-with-input.py", "", "helloWorld.log")
+    return endOfAllJobs(J)
+
+
+def helloWorld_input_single():
+    """simple hello world job with input"""
+
+    J = baseToAllJobs("helloWorld_input_single")
+    try:
+        J.setInputSandbox([find_all("exe-script-with-input-single-location.py", rootPath, "DIRAC/tests/Workflow")[0]])
+    except IndexError:
+        try:
+            J.setInputSandbox([find_all("exe-script-with-input-single-location.py", ".", "DIRAC/tests/Workflow")[0]])
+        except IndexError:  # we are in Jenkins
+            J.setInputSandbox(
+                [
+                    find_all(
+                        "exe-script-with-input-single-location.py", os.environ["WORKSPACE"], "DIRAC/tests/Workflow"
+                    )[0]
+                ]
+            )
+    J.setExecutable("exe-script-with-input-single-location.py", "", "helloWorld.log")
+    return endOfAllJobs(J)
+
+
 def helloWorldCERN():
     """simple hello world job to CERN"""
 
@@ -262,6 +299,28 @@ def parametricJob():
     return endOfAllJobs(J)
 
 
+def parametricJobInputData():
+    """Creates a parametric job with 3 subjobs which are simple hello world jobs, but with input data"""
+
+    J = baseToAllJobs("parametricJobInput")
+    try:
+        J.setInputSandbox([find_all("exe-script.py", rootPath, "DIRAC/tests/Workflow")[0]])
+    except IndexError:
+        try:
+            J.setInputSandbox([find_all("exe-script.py", ".", "DIRAC/tests/Workflow")[0]])
+        except IndexError:  # we are in Jenkins
+            J.setInputSandbox([find_all("exe-script.py", os.environ["WORKSPACE"], "DIRAC/tests/Workflow")[0]])
+    J.setParameterSequence("args", ["one", "two", "three"])
+    J.setParameterSequence("iargs", [1, 2, 3])
+    J.setParameterSequence(
+        "InputData",
+        ["/dteam/user/f/fstagni/test/1.txt", "/dteam/user/f/fstagni/test/2.txt", "/dteam/user/f/fstagni/test/3.txt"],
+    )
+    J.setInputDataPolicy("download")
+    J.setExecutable("exe-script.py", arguments=": testing %(args)s %(iargs)s", logFile="helloWorld_%n.log")
+    return endOfAllJobs(J)
+
+
 def jobWithOutput():
     """Creates a job that uploads an output.
     The output SE is not set here, so it would use the default /Resources/StorageElementGroups/SE-USER

tests/System/exe-script-with-input-single-location.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+
+"""Script to run Executable application"""
+from os import system
+import sys
+
+# Main
+if __name__ == "__main__":
+    sys.exit(int(system("""cat testInputFileSingleLocation.txt""") / 256))

tests/System/exe-script-with-input.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+
+"""Script to run Executable application"""
+import sys
+from os import system
+
+# Main
+if __name__ == "__main__":
+    sys.exit(int(system("""cat testInputFile.txt""") / 256))

tests/System/testInputFile.txt

@@ -0,0 +1 @@
+this is just a test

tests/System/testInputFileSingleLocation.txt

@@ -0,0 +1,3 @@
+this is just a test
+
+This file should be at /dteam/user/f/fstagni/test/testInputFileSingleLocation.txt in RAL-SE

tests/System/transformationSystem.sh

@@ -45,7 +45,7 @@ directory=/dteam/diracCertification/Test/INIT/$version/$tdate/$stime
 #selecting a random USER Storage Element
 #SEs=$(dirac-dms-show-se-status |grep USER |grep -v 'Banned\|Degraded\|-2' | awk '{print $1}')
 #get all SEs ending with -SE that are Active
-SEs=$(dirac-dms-show-se-status | grep -e "-SE \|-disk" | grep -v 'RAL\|CESNET-SE\|Banned\|Probing\|Error\|-new' | awk '{print $1}')
+SEs=$(dirac-dms-show-se-status | grep -e "-SE \|-disk" | grep -v 'RAL\|Banned\|Probing\|Error\|-new' | awk '{print $1}')
 
 x=0
 for n in $SEs

tests/System/transformation_replication.sh

@@ -45,7 +45,7 @@ echo "Creating TransformationSystemTest"
 mkdir -p TransformationSystemTest
 directory=/dteam/diracCertification/Test/INIT/$version/$tdate/$stime/replication
 #get all SEs ending with -SE that are Active, randomize, turn into array
-SEs=( $(dirac-dms-show-se-status | grep -e "-SE \|-disk " | grep -v 'RAL\|CESNET\|Banned\|Probing\|Error\|-new' | awk '{print $1}' | sort -R | xargs) )
+SEs=( $(dirac-dms-show-se-status | grep -e "-SE \|-disk " | grep -v 'RAL\|Banned\|Probing\|Error\|-new' | awk '{print $1}' | sort -R | xargs) )
 
 SOURCE_SE=${SEs[0]}
 TARGET_SE=${SEs[1]}

tests/System/unitTestUserJobs.py

@@ -1,8 +1,9 @@
 """ Collection of user jobs for testing purposes
 """
 # pylint: disable=wrong-import-position, invalid-name
-import unittest
+import sys
 import time
+import unittest
 
 import DIRAC
 
@@ -22,7 +23,29 @@ class GridSubmissionTestCase(unittest.TestCase):
     """Base class for the Regression test cases"""
 
     def setUp(self):
-        pass
+        result = getProxyInfo()
+        if result["Value"]["group"] not in ["dteam_user", "gridpp_user"]:
+            print("GET A USER GROUP")
+            sys.exit(1)
+
+        res = DataManager().getReplicas(
+            [
+                "/dteam/user/f/fstagni/test/testInputFileSingleLocation.txt",
+                "/dteam/user/f/fstagni/test/testInputFile.txt",
+            ]
+        )
+        if not res["OK"]:
+            print(f"DATAMANAGER.getRepicas failure: {res['Message']}")
+            sys.exit(1)
+        if res["Value"]["Failed"]:
+            print(f"DATAMANAGER.getRepicas failed for something: {res['Value']['Failed']}")
+            sys.exit(1)
+
+        replicas = res["Value"]["Successful"]
+        if list(replicas["/dteam/user/f/fstagni/test/testInputFile.txt"]) != ["RAL-SE", "UKI-LT2-IC-HEP-disk"]:
+            print("/dteam/user/f/fstagni/test/testInputFile.txt locations are not correct")
+        if list(replicas["/dteam/user/f/fstagni/test/testInputFileSingleLocation.txt"]) != ["RAL-SE"]:
+            print("/dteam/user/f/fstagni/test/testInputFileSingleLocation.txt locations are not correct")
 
     def tearDown(self):
         pass
@@ -37,6 +60,14 @@ def test_submit(self):
         self.assertTrue(res["OK"])
         jobsSubmittedList.append(res["Value"])
 
+        res = helloWorld_input()
+        self.assertTrue(res["OK"])
+        jobsSubmittedList.append(res["Value"])
+
+        res = helloWorld_input_single()
+        self.assertTrue(res["OK"])
+        jobsSubmittedList.append(res["Value"])
+
         res = helloWorldCERN()
         self.assertTrue(res["OK"])
         jobsSubmittedList.append(res["Value"])
@@ -77,6 +108,10 @@ def test_submit(self):
         self.assertTrue(res["OK"])
         jobsSubmittedList.append(res["Value"])
 
+        res = parametricJobInputData()
+        self.assertTrue(res["OK"])
+        jobsSubmittedList.append(res["Value"])
+
         res = jobWithOutput()
         self.assertTrue(res["OK"])
         jobsSubmittedList.append(res["Value"])