From d5a9fd2dd2a87a95fbfb199ee4ba1da4ed6e961c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Thu, 18 Jan 2024 11:47:15 +0100 Subject: [PATCH] Add some missing config defaults MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Aurélien Bompard --- mirrormanager2/default_config.py | 11 +++++++++++ mirrormanager2/lib/umdl.py | 17 ++++++----------- mirrormanager2/utility/crawler.py | 3 +-- mirrormanager2/utility/umdl2.py | 8 ++++---- .../utility/update_master_directory_list.py | 12 ++++++------ utility/mirrormanager2.cfg.sample | 15 +++++++++++++-- 6 files changed, 41 insertions(+), 25 deletions(-) diff --git a/mirrormanager2/default_config.py b/mirrormanager2/default_config.py index 37f4855e6..1f62c3ece 100644 --- a/mirrormanager2/default_config.py +++ b/mirrormanager2/default_config.py @@ -165,6 +165,15 @@ # in front of the application). CHECK_SESSION_IP = True +# Specify additional rsync parameters for the crawler +# # --timeout 14400: abort rsync crawl after 4 hours +# # --no-human-readable: because rsync made things pretty by default in 3.1.x +CRAWLER_RSYNC_PARAMETERS = "--no-motd" + +# This is a list of directories which MirrorManager will ignore while guessing +# the version and architecture from a path. +SKIP_PATHS_FOR_VERSION = [] + ### # Configuration options used by the utilities ### @@ -186,4 +195,6 @@ # Whether to use Fedora Messaging for notifications USE_FEDORA_MESSAGING = True +UMDL_PREFIX = "" + UMDL_MASTER_DIRECTORIES = [] diff --git a/mirrormanager2/lib/umdl.py b/mirrormanager2/lib/umdl.py index 087b981be..7347a804b 100644 --- a/mirrormanager2/lib/umdl.py +++ b/mirrormanager2/lib/umdl.py @@ -109,8 +109,7 @@ def setup_arch_version_cache(session): def guess_ver_arch_from_path(session, category, path, config=None): if config: dname = os.path.join(category.topdir.name, path) - skip_dirs = config.get("SKIP_PATHS_FOR_VERSION", []) - for skip in skip_dirs: + for skip in config["SKIP_PATHS_FOR_VERSION"]: if dname.startswith(skip): return (None, None) @@ -157,7 +156,7 @@ def make_file_details_from_checksums(session, config, relativeDName, D): def _parse_checksum_file(relativeDName, checksumlen): r = {} try: - f = open(os.path.join(config.get("UMDL_PREFIX", ""), relativeDName)) + f = open(os.path.join(config["UMDL_PREFIX"], relativeDName)) for line in f: line = line.strip() s = line.split() @@ -177,9 +176,7 @@ def _checksums_from_globs(config, relativeDName, globs, checksumlen): d = {} checksum_files = [] for g in globs: - checksum_files.extend( - glob.glob(os.path.join(config.get("UMDL_PREFIX", ""), relativeDName, g)) - ) + checksum_files.extend(glob.glob(os.path.join(config["UMDL_PREFIX"], relativeDName, g))) for f in checksum_files: d.update(_parse_checksum_file(f, checksumlen)) return d @@ -205,7 +202,7 @@ def _checksums_from_globs(config, relativeDName, globs, checksumlen): for f in files: try: - s = os.stat(os.path.join(config.get("UMDL_PREFIX", ""), relativeDName, f)) + s = os.stat(os.path.join(config["UMDL_PREFIX"], relativeDName, f)) except OSError: # bail if the file doesn't actually exist continue @@ -250,9 +247,7 @@ def make_repo_file_details(session, config, relativeDName, D, category, target): logger.warning(f"{warning}: {target!r} not in {allowed_targets!r}") return - absolutepath = os.path.join( - config.get("UMDL_PREFIX", ""), category.topdir.name, relativeDName, target - ) + absolutepath = os.path.join(config["UMDL_PREFIX"], category.topdir.name, relativeDName, target) if not os.path.exists(absolutepath): logger.warning(f"{warning}: {absolutepath!r} does not exist") @@ -399,7 +394,7 @@ def short_filelist(config, relativeDName, files): date_file_list = [] for k in files: try: - s = os.stat(os.path.join(config.get("UMDL_PREFIX", ""), relativeDName, k)) + s = os.stat(os.path.join(config["UMDL_PREFIX"], relativeDName, k)) except OSError: continue diff --git a/mirrormanager2/utility/crawler.py b/mirrormanager2/utility/crawler.py index a5ebf48bf..0ddf731ad 100755 --- a/mirrormanager2/utility/crawler.py +++ b/mirrormanager2/utility/crawler.py @@ -990,9 +990,8 @@ def try_per_category( local_timeout = int(remaining * 0.9) rsync_start_time = datetime.datetime.utcnow() - params = config.get("CRAWLER_RSYNC_PARAMETERS", "--no-motd") try: - result, listing = run_rsync(url, params, logger, local_timeout) + result, listing = run_rsync(url, config["CRAWLER_RSYNC_PARAMETERS"], logger, local_timeout) except Exception: logger.exception("Failed to run rsync.", exc_info=True) return False diff --git a/mirrormanager2/utility/umdl2.py b/mirrormanager2/utility/umdl2.py index 3c4df570c..2a5c34060 100755 --- a/mirrormanager2/utility/umdl2.py +++ b/mirrormanager2/utility/umdl2.py @@ -71,7 +71,7 @@ def sync_directories_from_fullfilelist(session, config, diskpath, category, excl if row in seen: continue seen.add(row) - relativeDName = os.path.join(diskpath, row).replace(config.get("UMDL_PREFIX", ""), "") + relativeDName = os.path.join(diskpath, row).replace(config["UMDL_PREFIX"], "") relativeDName = relativeDName.rstrip("/") logger.debug(" walking %r" % relativeDName) if is_excluded(relativeDName, stdexcludes + excludes): @@ -80,7 +80,7 @@ def sync_directories_from_fullfilelist(session, config, diskpath, category, excl # avoid disappearing directories try: - s = os.stat(os.path.join(config.get("UMDL_PREFIX", ""), relativeDName)) + s = os.stat(os.path.join(config["UMDL_PREFIX"], relativeDName)) ctime = s[stat.ST_CTIME] except OSError: logger.debug("Avoiding %r, dissappeared." % relativeDName) @@ -103,7 +103,7 @@ def sync_directories_from_disk(session, config, diskpath, category, excludes=Non diskpath = diskpath.rstrip("/") for dirpath, _dirnames, _filenames in os.walk(diskpath, topdown=True): - relativeDName = dirpath.replace(config.get("UMDL_PREFIX", ""), "") + relativeDName = dirpath.replace(config["UMDL_PREFIX"], "") logger.debug(" walking %r" % relativeDName) if is_excluded(relativeDName, stdexcludes + excludes): @@ -112,7 +112,7 @@ def sync_directories_from_disk(session, config, diskpath, category, excludes=Non # avoid disappearing directories try: - s = os.stat(os.path.join(config.get("UMDL_PREFIX", ""), relativeDName)) + s = os.stat(os.path.join(config["UMDL_PREFIX"], relativeDName)) ctime = s[stat.ST_CTIME] except OSError: logger.debug("Avoiding %r, dissappeared." % relativeDName) diff --git a/mirrormanager2/utility/update_master_directory_list.py b/mirrormanager2/utility/update_master_directory_list.py index 2bdff4393..465f19b50 100755 --- a/mirrormanager2/utility/update_master_directory_list.py +++ b/mirrormanager2/utility/update_master_directory_list.py @@ -116,7 +116,7 @@ def _handle_checksum_line(line, checksumlen): def _parse_checksum_file(relativeDName, checksumlen): r = {} try: - with open(os.path.join(config.get("UMDL_PREFIX", ""), relativeDName)) as f: + with open(os.path.join(config["UMDL_PREFIX"], relativeDName)) as f: for line in f: filename, checksum = _handle_checksum_line(line, checksumlen) if filename is not None: @@ -146,7 +146,7 @@ def _checksums_from_globs(relativeDName, globs, checksumlen): if re.compile(g).match(f): checksum_files.append(os.path.join(relativeDName, f)) else: - for f in os.listdir(os.path.join(config.get("UMDL_PREFIX", ""), relativeDName)): + for f in os.listdir(os.path.join(config["UMDL_PREFIX"], relativeDName)): if re.compile(g).match(f): checksum_files.append(os.path.join(relativeDName, f)) for f in checksum_files: @@ -184,7 +184,7 @@ def _checksums_from_globs(relativeDName, globs, checksumlen): continue else: try: - s = os.stat(os.path.join(config.get("UMDL_PREFIX", ""), D.name, f)) + s = os.stat(os.path.join(config["UMDL_PREFIX"], D.name, f)) except OSError: # bail if the file doesn't actually exist continue @@ -571,7 +571,7 @@ def sync_category_directory(session, config, category, relativeDName, ctime, fil # stating changed files to see if they are readable s = None try: - s = os.stat(os.path.join(config.get("UMDL_PREFIX", ""), topdir, relativeDName)) + s = os.stat(os.path.join(config["UMDL_PREFIX"], topdir, relativeDName)) except OSError: # The main reason for this execption is that the # file from the fullfiletimelist does not exist. @@ -871,7 +871,7 @@ def _handle_fedora_linux_category(path): # A set with a list of directories with a repository (repodata) repo = set() - umdl_prefix = config.get("UMDL_PREFIX", "") + umdl_prefix = config["UMDL_PREFIX"] # Not opening the file as stream and reading line by line as this breaks # if the file changes. As this can happen, the file is loaded once into # memory using mmap. @@ -912,7 +912,7 @@ def _handle_fedora_linux_category(path): # get all directories and their ctime tmp = os.path.join(diskpath, _handle_fedora_linux_category(col[3])) - tmp = tmp.replace(config.get("UMDL_PREFIX", ""), "") + tmp = tmp.replace(umdl_prefix, "") tmp = str(tmp, "utf8") ctimes[tmp] = int(col[0]) # Only if a directory contains a 'repodata' directory diff --git a/utility/mirrormanager2.cfg.sample b/utility/mirrormanager2.cfg.sample index f20be84b6..bd7382943 100644 --- a/utility/mirrormanager2.cfg.sample +++ b/utility/mirrormanager2.cfg.sample @@ -51,7 +51,7 @@ OIDC_CLIENT_SECRETS = "/etc/mirrormanager/client_secrets.json" # Email of the admin to which send notification or error ADMIN_EMAIL = "admin@fedoraproject.org" -# Email address used in the 'From' field of the emails sent. +# Email address used in the "From" field of the emails sent. # Default: ``nobody@fedoraproject.org``. #EMAIL_FROM = "nobody@fedoraproject.org" @@ -126,7 +126,7 @@ ADMIN_EMAIL = "admin@fedoraproject.org" # for host category URLs at all. # The following is the default for Fedora to exclude FTP based # mirrors to be added. Removing this confguration option -# or setting it to '' removes any protocol restrictions. +# or setting it to "" removes any protocol restrictions. #MM_PROTOCOL_REGEX = "^(?!ftp)(.*)$" # The netblock size parameters define which netblock sizes can be @@ -146,6 +146,15 @@ ADMIN_EMAIL = "admin@fedoraproject.org" # in front of the application). #CHECK_SESSION_IP = True +# Specify additional rsync parameters for the crawler +# # --timeout 14400: abort rsync crawl after 4 hours +# # --no-human-readable: because rsync made things pretty by default in 3.1.x +#CRAWLER_RSYNC_PARAMETERS = "--no-motd" + +# This is a list of directories which MirrorManager will ignore while guessing +# the version and architecture from a path. +#SKIP_PATHS_FOR_VERSION = [] + ### # Configuration options used by the utilities ### @@ -170,6 +179,8 @@ CRAWLER_SEND_EMAIL = True # Whether to use Fedora Messaging for notifications #USE_FEDORA_MESSAGING = True +UMDL_PREFIX = "/srv/" + UMDL_MASTER_DIRECTORIES = [ { 'type': 'directory',