Remove pkg_resources (#283)

* Replace deprecated pkg_resources with importlib.metadata

* Require a mongomock version that is free of pkg_resources

ampel/cli/JobCommand.py

@@ -8,6 +8,8 @@
 # Last Modified By:    valery brinnel <[email protected]>
 
 import filecmp
+import importlib
+import importlib.metadata
 import io
 import os
 import platform
@@ -23,7 +25,6 @@
 from time import sleep, time
 from typing import Any
 
-import pkg_resources
 import psutil
 import ujson
 import yaml
@@ -371,7 +372,7 @@ def Pool(**kwargs):
 			for k in config_dict['build']:
 				if 'ampel-' in k:
 					config_v = config_dict['build'][k]
-					current_v = pkg_resources.get_distribution(k).version
+					current_v = importlib.metadata.distribution(k).version
 					if (
 						config_v != current_v and
 						yes_no(

ampel/config/builder/ConfigBuilder.py

@@ -10,6 +10,7 @@
 import datetime
 import getpass
 import importlib
+import importlib.metadata
 import json
 import os
 import re
@@ -19,7 +20,6 @@
 from multiprocessing import Pool
 from typing import Any
 
-import pkg_resources
 import yaml
 
 from ampel.abstract.AbsChannelTemplate import AbsChannelTemplate
@@ -435,7 +435,7 @@ def build_config(self,
 		}
 
 		for k in ('ampel-interface', 'ampel-core'):
-			d['build'][k] = pkg_resources.get_distribution(k).version
+			d['build'][k] = importlib.metadata.distribution(k).version
 
 		dists: set[tuple[str, str | float | int]] = set()
 		for k in ('unit', 'channel'):

ampel/config/builder/DistConfigBuilder.py

@@ -7,23 +7,17 @@
 # Last Modified Date:  18.12.2022
 # Last Modified By:    valery brinnel <[email protected]>
 
-import json
-import os
 import re
 from collections.abc import Callable, Sequence
 from functools import partial
+from importlib import metadata
 from typing import Any
 
-import pkg_resources
 import yaml
-from pkg_resources import (  # type: ignore[attr-defined]
-	DistInfoDistribution,
-	EggInfoDistribution,
-)
 
 from ampel.config.builder.ConfigBuilder import ConfigBuilder
 from ampel.log import SHOUT, VERBOSE
-from ampel.util.distrib import get_dist_names, get_files
+from ampel.util.distrib import PathLike, PathList, get_dist_names, get_files
 
 
 class DistConfigBuilder(ConfigBuilder):
@@ -88,13 +82,13 @@ def load_distrib(self,
 		"""
 		try:
 
-			distrib = pkg_resources.get_distribution(dist_name)
+			distrib = metadata.distribution(dist_name)
 			all_conf_files = get_files(dist_name, conf_dir, re.compile(rf".*\.{ext}$"))
 
 			if all_conf_files and self.verbose:
 				self.logger.log(VERBOSE, "Following conf files will be parsed:")
 				for el in all_conf_files:
-					self.logger.log(VERBOSE, el)
+					self.logger.log(VERBOSE, el.as_posix())
 
 			if ampel_conf := self.get_conf_file(all_conf_files, f"ampel.{ext}"):
 				self.load_conf_using_func(
@@ -104,7 +98,7 @@ def load_distrib(self,
 			# Channel, mongo (and template) can be defined by multiple files
 			# in a directory named after the corresponding config section name
 			for key in ("channel", "mongo", "process"):
-				if specialized_conf_files := self.get_conf_files(all_conf_files, f"/{key}/"):
+				if specialized_conf_files := self.get_conf_files(all_conf_files, f"*/{key}/*"):
 					for f in specialized_conf_files:
 						self.load_conf_using_func(distrib, f, self.first_pass_config[key].add)
 
@@ -123,16 +117,16 @@ def load_distrib(self,
 					)
 
 			# Try to load templates from folder template (defined by 'Ampel-ZTF' for ex.)
-			if template_conf_files := self.get_conf_files(all_conf_files, "/template/"):
+			if template_conf_files := self.get_conf_files(all_conf_files, "*/template/*"):
 				for f in template_conf_files:
 					self.load_conf_using_func(distrib, f, self.register_templates)
 
 			# Try to load templates from template.conf
-			if template_conf := self.get_conf_file(all_conf_files, "/template.{ext}"):
+			if template_conf := self.get_conf_file(all_conf_files, "template.{ext}"):
 				self.load_conf_using_func(distrib, template_conf, self.register_templates)
 
 			if all_conf_files:
-				self.logger.info(f"Not all conf files were loaded from distribution '{distrib.project_name}'")
+				self.logger.info(f"Not all conf files were loaded from distribution '{distrib.name}'")
 				self.logger.info(f"Unprocessed conf files: {all_conf_files}")
 
 		except Exception as e:
@@ -163,8 +157,8 @@ def register_tier_conf(self,
 
 
 	def load_conf_using_func(self,
-		distrib: EggInfoDistribution | DistInfoDistribution,
-		file_rel_path: str,
+		distrib: metadata.Distribution,
+		file_rel_path: "PathLike",
 		func: Callable[[dict[str, Any], str, str, str], None],
 		raise_exc: bool = True,
 		**kwargs
@@ -174,25 +168,17 @@ def load_conf_using_func(self,
 
 			if self.verbose:
 				self.logger.log(VERBOSE,
-					f"Loading {file_rel_path} from distribution '{distrib.project_name}'"
+					f"Loading {file_rel_path} from distribution '{distrib.name}'"
 				)
 
-			if file_rel_path.endswith("json"):
-				load = json.loads
-			elif file_rel_path.endswith("yml") or file_rel_path.endswith("yaml"):
-				load = yaml.safe_load # type: ignore[assignment]
-
-			if os.path.isabs(file_rel_path):
-				with open(file_rel_path) as f:
-					payload = f.read()
-			else:
-				payload = distrib.get_resource_string(__name__, file_rel_path)
+			# NB: read both YAML and JSON (which is a subset of YAML)
+			content = yaml.safe_load(file_rel_path.read_text())
 
 			func(
-				load(payload),
-				distrib.project_name,
+				content,
+				distrib.name,
 				distrib.version,
-				file_rel_path,
+				file_rel_path.as_posix(),
 				**kwargs
 			)
 
@@ -201,96 +187,39 @@ def load_conf_using_func(self,
 				raise
 			self.error = True
 			self.logger.error(
-				f"File '{file_rel_path}' not found in distribution '{distrib.project_name}'"
+				f"File '{file_rel_path}' not found in distribution '{distrib.name}'"
 			)
 
 		except Exception as e:
 			if raise_exc:
 				raise
 			self.error = True
 			self.logger.error(
-				f"Error occured loading '{file_rel_path}' from distribution '{distrib.project_name}'",
-				exc_info=e
-			)
-
-
-	def load_tier_config_file(self,
-		distrib: EggInfoDistribution | DistInfoDistribution,
-		file_rel_path: str,
-		root_key: str
-	) -> None:
-		"""
-		Files loaded by this method must have the following JSON structure:
-		{
-			"t0": { ... },
-			"t1": { ... },
-			...
-		}
-		whereby the t0, t1, t2 and t3 keys are optional (at least one is required though)
-		"""
-
-		try:
-
-			if self.verbose:
-				self.logger.log(VERBOSE,
-					f"Loading {file_rel_path} from distribution '{distrib.project_name}'"
-				)
-
-			if file_rel_path.endswith("json"):
-				load = json.loads
-			elif file_rel_path.endswith("yml") or file_rel_path.endswith("yaml"):
-				load = yaml.safe_load # type: ignore[assignment]
-
-			d = load(
-				distrib.get_resource_string(__name__, file_rel_path)
-				if not os.path.isabs(file_rel_path)
-				else file_rel_path
-			)
-
-			for k in ("t0", "t1", "t2", "t3", "ops"):
-				if k in d:
-					self.first_pass_config[root_key][k].add(
-						d[k],
-						dist_name = distrib.project_name,
-						version = distrib.version,
-						register_file = file_rel_path,
-					)
-
-		except FileNotFoundError:
-			self.error = True
-			self.logger.error(
-				f"File '{file_rel_path}' not found in distribution '{distrib.project_name}'"
-			)
-
-		except Exception as e:
-			self.error = True
-			self.logger.error(
-				f"Error occured loading '{file_rel_path}' "
-				f"from distribution '{distrib.project_name}')",
+				f"Error occured loading '{file_rel_path}' from distribution '{distrib.name}'",
 				exc_info=e
 			)
 
 
 	@staticmethod
-	def get_conf_file(files: list[str], key: str) -> None | str:
+	def get_conf_file(files: "PathList", key: str) -> "None | PathLike":
 		"""
 		Extract the first entry who matches the provided 'key' from the provided list
 		Note: this method purposely modifies the input list (removes matched element)
 		"""
 		for el in files:
-			if key in el:
+			if el.match(key):
 				files.remove(el)
 				return el
 		return None
 
 
 	@staticmethod
-	def get_conf_files(files: list[str], key: str) -> list[str]:
+	def get_conf_files(files: "PathList", key: str) -> "PathList":
 		"""
 		Extract all entries who matches the provided 'key' from the provided list
 		Note: this method purposely modifies the input list (removes matched elements)
 		"""
-		ret = [el for el in files if key in el]
+		ret = [el for el in files if el.match(key)]
 		for el in ret:
 			files.remove(el)
 		return ret

ampel/config/collector/UnitConfigCollector.py

@@ -64,7 +64,7 @@ def add(self, # type: ignore[override]
 					if el.split('.')[-1][0].islower():
 						package_path = el.replace(".", sep)
 						try:
-							for fpath in get_files(dist_name, lookup_dir=package_path):
+							for fpath in map(str, get_files(dist_name, lookup_dir=package_path)):
 								if sep in fpath.replace(package_path + sep, ""):
 									continue
 								self.add(

ampel/test/test_config.py

@@ -159,7 +159,7 @@ def test_collect_bad_unit(tmp_path: Path, mocker: MockerFixture) -> None:
 
     mocker.patch(
         "ampel.config.builder.DistConfigBuilder.get_files",
-        return_value=[str(bad_unit_config)],
+        return_value=[bad_unit_config],
     )
 
     cb.load_distributions()