Considerably speed up the script 6s -> 1s

This was achieved by:
* Using ProcessPoolExecutor instead of ThreadPoolExecutor when
  creating each day. The latter was mistakenly used.
* Loading the github_languages.yaml with yaml.CLoader instead
  of yaml.FullLoader (speed-up: 0.5s -> 0.1s)
* Removed checking of each file separately if it is git-ignored.
  Now just iterate over all git-tracked files.

README.md

@@ -75,7 +75,7 @@ with examples showing how it changes the look.
 |----------------------------------------------------------|---------------------------|--------------------------------|
 | `time_and_rank` if cookie is available, else `checkmark` | ![](examples/01basic.png) | ![](examples/01checkmarks.png) |
 
-* `--contrast_improvement_type=`:
+* `--contrast-improvement-type=`:
 
 | `outline` (default)          | `dark`                 | `none`                 |
 |-----------------------------|--------------------------|--------------------------|

aoc_tiles/cli.py

@@ -6,6 +6,7 @@
 from typing import Literal, Union, get_args, List, get_origin
 
 import rich.traceback
+from loguru import logger
 
 from aoc_tiles.config import Config
 from aoc_tiles.make_tiles import TileMaker
@@ -81,9 +82,13 @@ def cli_parse_config(datacls):
 
 
 def main():
+    logger.debug("Starting AOC Tiles")
     rich.traceback.install()
+    logger.debug("Rich installed")
     config = cli_parse_config(Config)
+    logger.debug("Config parsed")
     TileMaker(config).make_tiles()
+    logger.debug("Tiles made")
 
 
 if __name__ == "__main__":

aoc_tiles/colors.py

@@ -4,6 +4,7 @@
 from typing import Tuple, Dict
 
 import yaml
+from loguru import logger
 
 GITHUB_LANGUAGES_PATH = Path(__file__).parent / "resources" / "github_languages.yaml"
 
@@ -15,7 +16,12 @@
 def extension_to_colors() -> Dict[str, str]:
     extension_to_color = {}
     with open(GITHUB_LANGUAGES_PATH) as file:
-        github_languages = yaml.load(file, Loader=yaml.FullLoader)
+        logger.debug("Loading github_languages.yaml from {}", GITHUB_LANGUAGES_PATH)
+        yaml_loader = yaml.CLoader if yaml.__with_libyaml__ else yaml.Loader
+        if not yaml.__with_libyaml__:
+            logger.warning("Using slow yaml parser (0.5s vs 0.1s)!")
+        github_languages = yaml.load(file, Loader=yaml_loader)
+        logger.debug("Loaded github_languages.yaml from {}", GITHUB_LANGUAGES_PATH)
         for language, data in github_languages.items():
             if "color" in data and "extensions" in data and data["type"] == "programming" and language not in excludes:
                 for extension in data["extensions"]:

aoc_tiles/make_tiles.py

@@ -1,11 +1,8 @@
-import concurrent
-import json
 import re
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import ProcessPoolExecutor
 from dataclasses import dataclass
 from pathlib import Path
-from pprint import pprint
-from typing import Dict, Set, List, Optional
+from typing import Dict, Set, List, Optional, Any
 
 from loguru import logger
 
@@ -73,6 +70,7 @@ def compose_solve_data(self) -> SolveData:
             day_to_solution = solution_paths_by_year.get(year, {})
             day_to_scores = {}
             if is_leaderboard_needed:
+                logger.debug("Requesting leaderboard for year {}", year)
                 day_to_scores = request_leaderboard(year, self.config)
 
             day_to_stars = {}
@@ -123,9 +121,6 @@ def handle_year(self, year: int, year_data: YearData):
         html = HTML()
         with html.tag("h1", align="center"):
             stars = sum(year_data.day_to_stars.values())
-            # stars = sum(
-            #     (ds.time1 is not None) + (ds.time2 is not None) for ds in leaderboard.values() if ds is not None
-            # )
             html.push(f"{year} - {stars} ⭐")
         max_solved_day = max(day for day, stars in year_data.day_to_stars.items() if stars > 0)
         max_day = 25 if self.config.create_all_days else max_solved_day
@@ -138,7 +133,7 @@ def handle_year(self, year: int, year_data: YearData):
         #         completed_solutions = {int(day): solutions for day, solutions in json.load(file).items()}
 
         day_to_future = {}
-        with ThreadPoolExecutor() as executor:
+        with ProcessPoolExecutor() as executor:
             for day in range(1, max_day + 1):
                 solutions = day_to_solutions.get(day, [])
                 stars = year_data.day_to_stars[day]

aoc_tiles/solutions.py

@@ -1,5 +1,6 @@
 import re
 from collections import defaultdict
+from functools import cache, lru_cache
 from pathlib import Path
 from pprint import pprint
 from typing import List, Dict, Optional
@@ -31,6 +32,7 @@ def get_solution_paths_by_year(self, aoc_dir: Path) -> Dict[Optional[int], Dict[
         day_to_solution_paths = defaultdict(lambda: defaultdict(list))
         year_pattern = re.compile(self.config.year_pattern)
         day_pattern = re.compile(self.config.day_pattern)
+        logger.debug("Finding solution files recursively in {}", aoc_dir)
         candidate_paths = sorted(self._find_recursive_solution_files(aoc_dir))
         logger.debug("Candidate paths: {}", candidate_paths)
         for path in candidate_paths:
@@ -59,17 +61,21 @@ def sort_key(path: Path):
         return solution_paths_dict
 
     def _find_recursive_solution_files(self, directory: Path) -> List[Path]:
+        if self.config.only_use_solutions_in_git:
+            files = [Path(s) for s in self.git_get_tracked_files()]
+        else:
+            files = directory.rglob("*")
+
+        logger.debug("Found {} files", len(files))
         solution_paths = []
-        for path in directory.rglob("*"):
-            # Either we don't care about git ignores, or we care and then check if file is ignored
-            not_git_ignored = not self.config.only_use_solutions_in_git
-            not_git_ignored |= self.config.only_use_solutions_in_git and self.git_is_file_tracked(path)
+        for path in files:
             extension_is_supported = path.suffix in extension_to_colors()
-            path_is_not_excluded = not any([path.match(exclude) for exclude in self.config.exclude_patterns])
-            if not path_is_not_excluded:
+            path_is_excluded = any([path.match(exclude) for exclude in self.config.exclude_patterns])
+            if path_is_excluded:
                 logger.debug("Excluded: {} because of patterns: {}", path, self.config.exclude_patterns)
-            if path.is_file() and extension_is_supported and not_git_ignored and path_is_not_excluded:
+            if path.is_file() and extension_is_supported and not path_is_excluded:
                 solution_paths.append(path)
+        logger.debug("Found {} solution files", len(solution_paths))
         return solution_paths
 
     def git_is_file_ignored(self, filepath):
@@ -79,6 +85,7 @@ def git_is_file_ignored(self, filepath):
         except GitCommandError:
             return False
 
+    @lru_cache
     def git_get_tracked_files(self) -> List[str]:
         return self.repository.git.ls_files().split('\n')
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "aoc-tiles"
-version = "0.4.0"
+version = "0.4.1"
 description = "Fancy Advent of Code README tiles showing the completion status of the challenges per day"
 authors = ["Brutenis Gliwa, <[email protected]>"]
 license = "Apache-2.0"