From 0674b665276ad2982e249dd6c9b4d1c09528dfd7 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Tue, 28 Feb 2023 17:24:42 -0600 Subject: [PATCH 1/4] OSDF cache JSON: don't list inactive or disabled caches (SOFTWARE-5440) --- src/stashcache.py | 2 +- src/webapp/topology.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/stashcache.py b/src/stashcache.py index 95e6ff728..82760cd51 100644 --- a/src/stashcache.py +++ b/src/stashcache.py @@ -556,7 +556,7 @@ def _namespace_dict(ns: Namespace): for group in resource_groups: for resource in group.resources: - if _resource_has_cache(resource): + if _resource_has_cache(resource) and resource.is_active: cache_resource_objs[resource.name] = resource cache_resource_dicts[resource.name] = _cache_resource_dict(resource) diff --git a/src/webapp/topology.py b/src/webapp/topology.py index 0f6cc85de..4c3b11b37 100644 --- a/src/webapp/topology.py +++ b/src/webapp/topology.py @@ -259,6 +259,11 @@ def get_tree(self, authorized=False, filters: Filters = None) -> Optional[Ordere return new_res + @property + def is_active(self): + """Check if the Resource is active and not disabled""" + return self.data.get("Active", True) and not self.data.get("Disable", False) + @property def is_ccstar(self): """Check if this site is tagged as a CC* Site""" From 93814159980cf62bb5db8579f7ac4a20d19b7a48 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Tue, 28 Feb 2023 17:38:23 -0600 Subject: [PATCH 2/4] OSDF cache JSON: don't list caches currently in downtime (SOFTWARE-5471) --- src/stashcache.py | 12 +++++++++--- src/webapp/topology.py | 6 +++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/stashcache.py b/src/stashcache.py index 82760cd51..76fb8668e 100644 --- a/src/stashcache.py +++ b/src/stashcache.py @@ -543,12 +543,15 @@ def _namespace_dict(ns: Namespace): } for cache_name, cache_resource_obj in cache_resource_objs.items(): - if resource_allows_namespace(cache_resource_obj, ns) and namespace_allows_cache_resource(ns, cache_resource_obj): + if resource_allows_namespace(cache_resource_obj, ns) and namespace_allows_cache_resource(ns, + cache_resource_obj): nsdict["caches"].append(cache_resource_dicts[cache_name]) return nsdict + # End helper functions - resource_groups: List[ResourceGroup] = global_data.get_topology().get_resource_group_list() + topology = global_data.get_topology() + resource_groups: List[ResourceGroup] = topology.get_resource_group_list() vos_data = global_data.get_vos_data() cache_resource_objs = {} # type: Dict[str, Resource] @@ -556,7 +559,10 @@ def _namespace_dict(ns: Namespace): for group in resource_groups: for resource in group.resources: - if _resource_has_cache(resource) and resource.is_active: + if (_resource_has_cache(resource) + and resource.is_active + and resource.name not in topology.downed_resource_names + ): cache_resource_objs[resource.name] = resource cache_resource_dicts[resource.name] = _cache_resource_dict(resource) diff --git a/src/webapp/topology.py b/src/webapp/topology.py index 4c3b11b37..aa1b61102 100644 --- a/src/webapp/topology.py +++ b/src/webapp/topology.py @@ -467,7 +467,8 @@ def __init__(self, rg: ResourceGroup, yaml_data: ParsedYaml, common_data: Common self.created_time = None if not is_null(yaml_data, "CreatedTime"): self.created_time = self.parsetime(yaml_data["CreatedTime"]) - self.res = rg.resources_by_name[yaml_data["ResourceName"]] + self.res_name = yaml_data["ResourceName"] + self.res = rg.resources_by_name[self.res_name] self.service_names = yaml_data["Services"] self.service_ids = [common_data.service_types[x] for x in yaml_data["Services"]] self.id = yaml_data["ID"] @@ -643,6 +644,7 @@ def __init__(self, common_data: CommonData): self.service_names_by_resource = {} # type: Dict[str, List[str]] self.downtime_path_by_resource_group = defaultdict(set) self.downtime_path_by_resource = {} + self.downed_resource_names = set() # resource names that are currently down def add_rg(self, facility_name: str, site_name: str, name: str, parsed_data: ParsedYaml): try: @@ -747,6 +749,8 @@ def add_downtime(self, sitename: str, rgname: str, downtime: ParsedYaml): log.warning("Invalid or missing data in downtime -- skipping: %r", err) return self.downtimes_by_timeframe[dt.timeframe].append(dt) + if dt.timeframe == Timeframe.PRESENT: + self.downed_resource_names.add(dt.res_name) def safe_get_resource_by_fqdn(self, fqdn: str) -> Optional[Resource]: """Returns the first resource that has the given FQDN or None if no such resource exists.""" From f73890f9d84e8566c609aa11b5ed2c385f830ae1 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Tue, 28 Feb 2023 18:09:17 -0600 Subject: [PATCH 3/4] Only exclude resources if the cache service itself is down (SOFTWARE-5471) --- src/stashcache.py | 14 +++++++++++++- src/webapp/topology.py | 4 ++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/stashcache.py b/src/stashcache.py index 76fb8668e..1dfa00449 100644 --- a/src/stashcache.py +++ b/src/stashcache.py @@ -548,6 +548,18 @@ def _namespace_dict(ns: Namespace): nsdict["caches"].append(cache_resource_dicts[cache_name]) return nsdict + def _resource_has_downed_cache(r: Resource, t: Topology): + if r.name not in t.present_downtimes_by_resource: + return False + downtimes = t.present_downtimes_by_resource[r.name] + for dt in downtimes: + try: + if XROOTD_CACHE_SERVER in dt.service_names: + return True + except (KeyError, AttributeError): + continue + return False + # End helper functions topology = global_data.get_topology() @@ -561,7 +573,7 @@ def _namespace_dict(ns: Namespace): for resource in group.resources: if (_resource_has_cache(resource) and resource.is_active - and resource.name not in topology.downed_resource_names + and not _resource_has_downed_cache(resource, topology) ): cache_resource_objs[resource.name] = resource cache_resource_dicts[resource.name] = _cache_resource_dict(resource) diff --git a/src/webapp/topology.py b/src/webapp/topology.py index aa1b61102..3743beeed 100644 --- a/src/webapp/topology.py +++ b/src/webapp/topology.py @@ -644,7 +644,7 @@ def __init__(self, common_data: CommonData): self.service_names_by_resource = {} # type: Dict[str, List[str]] self.downtime_path_by_resource_group = defaultdict(set) self.downtime_path_by_resource = {} - self.downed_resource_names = set() # resource names that are currently down + self.present_downtimes_by_resource = defaultdict(list) # type: defaultdict[str, List[Downtime]] def add_rg(self, facility_name: str, site_name: str, name: str, parsed_data: ParsedYaml): try: @@ -750,7 +750,7 @@ def add_downtime(self, sitename: str, rgname: str, downtime: ParsedYaml): return self.downtimes_by_timeframe[dt.timeframe].append(dt) if dt.timeframe == Timeframe.PRESENT: - self.downed_resource_names.add(dt.res_name) + self.present_downtimes_by_resource[dt.res_name].append(dt) def safe_get_resource_by_fqdn(self, fqdn: str) -> Optional[Resource]: """Returns the first resource that has the given FQDN or None if no such resource exists.""" From 68ff9c64f4aabbfb30d080c6b6e35a940c533373 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Wed, 1 Mar 2023 16:32:25 -0600 Subject: [PATCH 4/4] Fix weird indent --- src/stashcache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stashcache.py b/src/stashcache.py index 1dfa00449..64572ad6a 100644 --- a/src/stashcache.py +++ b/src/stashcache.py @@ -543,8 +543,8 @@ def _namespace_dict(ns: Namespace): } for cache_name, cache_resource_obj in cache_resource_objs.items(): - if resource_allows_namespace(cache_resource_obj, ns) and namespace_allows_cache_resource(ns, - cache_resource_obj): + if (resource_allows_namespace(cache_resource_obj, ns) and + namespace_allows_cache_resource(ns, cache_resource_obj)): nsdict["caches"].append(cache_resource_dicts[cache_name]) return nsdict