feat: add more SD tests

aldbr · Nov 16, 2023 · 55c4016 · 55c4016
1 parent 755d42b
commit 55c4016
Show file tree

Hide file tree

Showing 16 changed files with 528 additions and 383 deletions.
diff --git a/src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py b/src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py
@@ -255,10 +255,7 @@ def getQueues(siteList=None, ceList=None, ceTypeList=None, community=None, tags=
         for site in sites:
             if siteList and site not in siteList:
                 continue
-            if community:
-                comList = gConfig.getValue(f"/Resources/Sites/{grid}/{site}/VO", [])
-                if comList and community.lower() not in [cl.lower() for cl in comList]:
-                    continue
+
             siteCEParameters = {}
             result = gConfig.getOptionsDict(f"/Resources/Sites/{grid}/{site}/CEs")
             if result["OK"]:
@@ -275,10 +272,7 @@ def getQueues(siteList=None, ceList=None, ceTypeList=None, community=None, tags=
                         continue
                 if ceList and ce not in ceList:
                     continue
-                if community:
-                    comList = gConfig.getValue(f"/Resources/Sites/{grid}/{site}/CEs/{ce}/VO", [])
-                    if comList and community.lower() not in [cl.lower() for cl in comList]:
-                        continue
+
                 ceOptionsDict = dict(siteCEParameters)
                 result = gConfig.getOptionsDict(f"/Resources/Sites/{grid}/{site}/CEs/{ce}")
                 if not result["OK"]:
@@ -290,9 +284,23 @@ def getQueues(siteList=None, ceList=None, ceTypeList=None, community=None, tags=
                 queues = result["Value"]
                 for queue in queues:
                     if community:
-                        comList = gConfig.getValue(f"/Resources/Sites/{grid}/{site}/CEs/{ce}/Queues/{queue}/VO", [])
+                        # Community can be defined on site, CE or queue level
+                        paths = [
+                            f"/Resources/Sites/{grid}/{site}/CEs/{ce}/Queues/{queue}/VO",
+                            f"/Resources/Sites/{grid}/{site}/CEs/{ce}/VO",
+                            f"/Resources/Sites/{grid}/{site}/VO",
+                        ]
+
+                        # Try each path in order, stopping when we find a non-empty list
+                        for path in paths:
+                            comList = gConfig.getValue(path, [])
+                            if comList:
+                                break
+
+                        # If we found a list and the community is not in it, skip this iteration
                         if comList and community.lower() not in [cl.lower() for cl in comList]:
                             continue
+
                     if tags:
                         queueTags = gConfig.getValue(f"/Resources/Sites/{grid}/{site}/CEs/{ce}/Queues/{queue}/Tag", [])
                         queueTags = set(ceTags + queueTags)

diff --git a/src/DIRAC/Resources/Computing/AREXComputingElement.py b/src/DIRAC/Resources/Computing/AREXComputingElement.py
@@ -618,7 +618,11 @@ def getCEStatus(self):
             self.log.error("Failed getting the status of the CE.", result["Message"])
             return S_ERROR("Failed getting the status of the CE")
         response = result["Value"]
-        ceData = response.json()
+        try:
+            ceData = response.json()
+        except requests.JSONDecodeError:
+            self.log.exception("Failed decoding the status of the CE")
+            return S_ERROR(f"Failed decoding the status of the CE")
 
         # Look only in the relevant section out of the headache
         queueInfo = ceData["Domains"]["AdminDomain"]["Services"]["ComputingService"]["ComputingShare"]

diff --git a/src/DIRAC/Resources/Computing/CloudComputingElement.py b/src/DIRAC/Resources/Computing/CloudComputingElement.py
@@ -355,7 +355,7 @@ def _getMetadata(self, executableFile, pilotStamp=""):
             template = yaml.safe_load(template_fd)
         for filedef in template["write_files"]:
             if filedef["content"] == "PROXY_STR":
-                filedef["content"] = self.proxy
+                filedef["content"] = self.proxy.dumpAllToString()["Value"]
             elif filedef["content"] == "EXECUTABLE_STR":
                 filedef["content"] = exe_str
             elif "STAMP_STR" in filedef["content"]:
@@ -398,11 +398,7 @@ def _renewCloudProxy(self):
         if not res["OK"]:
             self.log.error("Could not download proxy", res["Message"])
             return False
-        resdump = res["Value"].dumpAllToString()
-        if not resdump["OK"]:
-            self.log.error("Failed to dump proxy to string", resdump["Message"])
-            return False
-        self.proxy = resdump["Value"]
+        self.proxy = res["Value"]
         self.valid = datetime.datetime.utcnow() + proxyLifetime * datetime.timedelta(seconds=1)
         return True
 

diff --git a/src/DIRAC/Resources/Computing/ComputingElement.py b/src/DIRAC/Resources/Computing/ComputingElement.py
@@ -263,6 +263,7 @@ def available(self):
         self.log.verbose("Max Number of Jobs:", maxTotalJobs)
         self.log.verbose("Max Waiting Jobs:", maxWaitingJobs)
 
+        result["CEInfoDict"] = ceInfoDict
         # If we reached the maximum number of total jobs, then the CE is not available
         totalJobs = runningJobs + waitingJobs
         if totalJobs >= maxTotalJobs:
@@ -283,8 +284,6 @@ def available(self):
         if availableProcessors is not None:
             additionalJobs = min(additionalJobs, availableProcessors)
         result["Value"] = additionalJobs
-
-        result["CEInfoDict"] = ceInfoDict
         return result
 
     #############################################################################