Add some missing config defaults

Signed-off-by: Aurélien Bompard <[email protected]>

mirrormanager2/default_config.py

@@ -165,6 +165,15 @@
 # in front of the application).
 CHECK_SESSION_IP = True
 
+# Specify additional rsync parameters for the crawler
+# # --timeout 14400: abort rsync crawl after 4 hours
+# # --no-human-readable: because rsync made things pretty by default in 3.1.x
+CRAWLER_RSYNC_PARAMETERS = "--no-motd"
+
+# This is a list of directories which MirrorManager will ignore while guessing
+# the version and architecture from a path.
+SKIP_PATHS_FOR_VERSION = []
+
 ###
 # Configuration options used by the utilities
 ###
@@ -186,4 +195,6 @@
 # Whether to use Fedora Messaging for notifications
 USE_FEDORA_MESSAGING = True
 
+UMDL_PREFIX = ""
+
 UMDL_MASTER_DIRECTORIES = []

mirrormanager2/lib/umdl.py

@@ -109,8 +109,7 @@ def setup_arch_version_cache(session):
 def guess_ver_arch_from_path(session, category, path, config=None):
     if config:
         dname = os.path.join(category.topdir.name, path)
-        skip_dirs = config.get("SKIP_PATHS_FOR_VERSION", [])
-        for skip in skip_dirs:
+        for skip in config["SKIP_PATHS_FOR_VERSION"]:
             if dname.startswith(skip):
                 return (None, None)
 
@@ -157,7 +156,7 @@ def make_file_details_from_checksums(session, config, relativeDName, D):
     def _parse_checksum_file(relativeDName, checksumlen):
         r = {}
         try:
-            f = open(os.path.join(config.get("UMDL_PREFIX", ""), relativeDName))
+            f = open(os.path.join(config["UMDL_PREFIX"], relativeDName))
             for line in f:
                 line = line.strip()
                 s = line.split()
@@ -177,9 +176,7 @@ def _checksums_from_globs(config, relativeDName, globs, checksumlen):
         d = {}
         checksum_files = []
         for g in globs:
-            checksum_files.extend(
-                glob.glob(os.path.join(config.get("UMDL_PREFIX", ""), relativeDName, g))
-            )
+            checksum_files.extend(glob.glob(os.path.join(config["UMDL_PREFIX"], relativeDName, g)))
         for f in checksum_files:
             d.update(_parse_checksum_file(f, checksumlen))
         return d
@@ -205,7 +202,7 @@ def _checksums_from_globs(config, relativeDName, globs, checksumlen):
 
     for f in files:
         try:
-            s = os.stat(os.path.join(config.get("UMDL_PREFIX", ""), relativeDName, f))
+            s = os.stat(os.path.join(config["UMDL_PREFIX"], relativeDName, f))
         except OSError:
             # bail if the file doesn't actually exist
             continue
@@ -250,9 +247,7 @@ def make_repo_file_details(session, config, relativeDName, D, category, target):
         logger.warning(f"{warning}: {target!r} not in {allowed_targets!r}")
         return
 
-    absolutepath = os.path.join(
-        config.get("UMDL_PREFIX", ""), category.topdir.name, relativeDName, target
-    )
+    absolutepath = os.path.join(config["UMDL_PREFIX"], category.topdir.name, relativeDName, target)
 
     if not os.path.exists(absolutepath):
         logger.warning(f"{warning}: {absolutepath!r} does not exist")
@@ -399,7 +394,7 @@ def short_filelist(config, relativeDName, files):
         date_file_list = []
         for k in files:
             try:
-                s = os.stat(os.path.join(config.get("UMDL_PREFIX", ""), relativeDName, k))
+                s = os.stat(os.path.join(config["UMDL_PREFIX"], relativeDName, k))
             except OSError:
                 continue
 

mirrormanager2/utility/crawler.py

@@ -990,9 +990,8 @@ def try_per_category(
     local_timeout = int(remaining * 0.9)
 
     rsync_start_time = datetime.datetime.utcnow()
-    params = config.get("CRAWLER_RSYNC_PARAMETERS", "--no-motd")
     try:
-        result, listing = run_rsync(url, params, logger, local_timeout)
+        result, listing = run_rsync(url, config["CRAWLER_RSYNC_PARAMETERS"], logger, local_timeout)
     except Exception:
         logger.exception("Failed to run rsync.", exc_info=True)
         return False

mirrormanager2/utility/umdl2.py

@@ -71,7 +71,7 @@ def sync_directories_from_fullfilelist(session, config, diskpath, category, excl
             if row in seen:
                 continue
             seen.add(row)
-            relativeDName = os.path.join(diskpath, row).replace(config.get("UMDL_PREFIX", ""), "")
+            relativeDName = os.path.join(diskpath, row).replace(config["UMDL_PREFIX"], "")
             relativeDName = relativeDName.rstrip("/")
             logger.debug("  walking %r" % relativeDName)
             if is_excluded(relativeDName, stdexcludes + excludes):
@@ -80,7 +80,7 @@ def sync_directories_from_fullfilelist(session, config, diskpath, category, excl
 
             # avoid disappearing directories
             try:
-                s = os.stat(os.path.join(config.get("UMDL_PREFIX", ""), relativeDName))
+                s = os.stat(os.path.join(config["UMDL_PREFIX"], relativeDName))
                 ctime = s[stat.ST_CTIME]
             except OSError:
                 logger.debug("Avoiding %r, dissappeared." % relativeDName)
@@ -103,7 +103,7 @@ def sync_directories_from_disk(session, config, diskpath, category, excludes=Non
     diskpath = diskpath.rstrip("/")
 
     for dirpath, _dirnames, _filenames in os.walk(diskpath, topdown=True):
-        relativeDName = dirpath.replace(config.get("UMDL_PREFIX", ""), "")
+        relativeDName = dirpath.replace(config["UMDL_PREFIX"], "")
         logger.debug("  walking %r" % relativeDName)
 
         if is_excluded(relativeDName, stdexcludes + excludes):
@@ -112,7 +112,7 @@ def sync_directories_from_disk(session, config, diskpath, category, excludes=Non
 
         # avoid disappearing directories
         try:
-            s = os.stat(os.path.join(config.get("UMDL_PREFIX", ""), relativeDName))
+            s = os.stat(os.path.join(config["UMDL_PREFIX"], relativeDName))
             ctime = s[stat.ST_CTIME]
         except OSError:
             logger.debug("Avoiding %r, dissappeared." % relativeDName)

mirrormanager2/utility/update_master_directory_list.py

@@ -116,7 +116,7 @@ def _handle_checksum_line(line, checksumlen):
     def _parse_checksum_file(relativeDName, checksumlen):
         r = {}
         try:
-            with open(os.path.join(config.get("UMDL_PREFIX", ""), relativeDName)) as f:
+            with open(os.path.join(config["UMDL_PREFIX"], relativeDName)) as f:
                 for line in f:
                     filename, checksum = _handle_checksum_line(line, checksumlen)
                     if filename is not None:
@@ -146,7 +146,7 @@ def _checksums_from_globs(relativeDName, globs, checksumlen):
                     if re.compile(g).match(f):
                         checksum_files.append(os.path.join(relativeDName, f))
             else:
-                for f in os.listdir(os.path.join(config.get("UMDL_PREFIX", ""), relativeDName)):
+                for f in os.listdir(os.path.join(config["UMDL_PREFIX"], relativeDName)):
                     if re.compile(g).match(f):
                         checksum_files.append(os.path.join(relativeDName, f))
         for f in checksum_files:
@@ -184,7 +184,7 @@ def _checksums_from_globs(relativeDName, globs, checksumlen):
                 continue
         else:
             try:
-                s = os.stat(os.path.join(config.get("UMDL_PREFIX", ""), D.name, f))
+                s = os.stat(os.path.join(config["UMDL_PREFIX"], D.name, f))
             except OSError:
                 # bail if the file doesn't actually exist
                 continue
@@ -571,7 +571,7 @@ def sync_category_directory(session, config, category, relativeDName, ctime, fil
         # stating changed files to see if they are readable
         s = None
         try:
-            s = os.stat(os.path.join(config.get("UMDL_PREFIX", ""), topdir, relativeDName))
+            s = os.stat(os.path.join(config["UMDL_PREFIX"], topdir, relativeDName))
         except OSError:
             # The main reason for this execption is that the
             # file from the fullfiletimelist does not exist.
@@ -871,7 +871,7 @@ def _handle_fedora_linux_category(path):
     # A set with a list of directories with a repository (repodata)
     repo = set()
 
-    umdl_prefix = config.get("UMDL_PREFIX", "")
+    umdl_prefix = config["UMDL_PREFIX"]
     # Not opening the file as stream and reading line by line as this breaks
     # if the file changes. As this can happen, the file is loaded once into
     # memory using mmap.
@@ -912,7 +912,7 @@ def _handle_fedora_linux_category(path):
                     # get all directories and their ctime
                     tmp = os.path.join(diskpath, _handle_fedora_linux_category(col[3]))
 
-                    tmp = tmp.replace(config.get("UMDL_PREFIX", ""), "")
+                    tmp = tmp.replace(umdl_prefix, "")
                     tmp = str(tmp, "utf8")
                     ctimes[tmp] = int(col[0])
                     # Only if a directory contains a 'repodata' directory

utility/mirrormanager2.cfg.sample

@@ -51,7 +51,7 @@ OIDC_CLIENT_SECRETS = "/etc/mirrormanager/client_secrets.json"
 # Email of the admin to which send notification or error
 ADMIN_EMAIL = "[email protected]"
 
-# Email address used in the 'From' field of the emails sent.
+# Email address used in the "From" field of the emails sent.
 # Default: ``[email protected]``.
 #EMAIL_FROM = "[email protected]"
 
@@ -126,7 +126,7 @@ ADMIN_EMAIL = "[email protected]"
 # for host category URLs at all.
 # The following is the default for Fedora to exclude FTP based
 # mirrors to be added. Removing this confguration option
-# or setting it to '' removes any protocol restrictions.
+# or setting it to "" removes any protocol restrictions.
 #MM_PROTOCOL_REGEX = "^(?!ftp)(.*)$"
 
 # The netblock size parameters define which netblock sizes can be
@@ -146,6 +146,15 @@ ADMIN_EMAIL = "[email protected]"
 # in front of the application).
 #CHECK_SESSION_IP = True
 
+# Specify additional rsync parameters for the crawler
+# # --timeout 14400: abort rsync crawl after 4 hours
+# # --no-human-readable: because rsync made things pretty by default in 3.1.x
+#CRAWLER_RSYNC_PARAMETERS = "--no-motd"
+
+# This is a list of directories which MirrorManager will ignore while guessing
+# the version and architecture from a path.
+#SKIP_PATHS_FOR_VERSION = []
+
 ###
 # Configuration options used by the utilities
 ###
@@ -170,6 +179,8 @@ CRAWLER_SEND_EMAIL =  True
 # Whether to use Fedora Messaging for notifications
 #USE_FEDORA_MESSAGING = True
 
+UMDL_PREFIX = "/srv/"
+
 UMDL_MASTER_DIRECTORIES = [
     {
         'type': 'directory',