Merge remote-tracking branch 'origin/master' into update-military-bur…

…den-gmsd

.git-blame-ignore-revs

@@ -9,3 +9,6 @@ ab772adce9a987639cb1cde15b9a62a349568582
 
 # switching to ruff
 5647751506f0685c6704aebc31a59632904d0741
+
+# update ruff
+38d649354920a5bd549a4e1ac02185882d13aa60

.vscode/settings.json

@@ -37,17 +37,11 @@
     "**/docs/architecture/*.md"
   ],
   "files.exclude": {
-    "etl/steps/archive": true,
-    "snapshots/archive": true,
     "**/dataset_*_config.json": true,
     "**/dataset_*_values.json": true,
     "**/dataset_*.json.dvc": true,
     "**/dataset_*.feather.dvc": true
   },
-  "search.exclude": {
-    "etl/steps/archive": true,
-    "snapshots/archive": true
-  },
   "yaml.format.printWidth": 999,
   "ruff.path": [
     ".venv/bin/ruff"

apps/chart_animation/cli.py

@@ -1,6 +1,5 @@
-"""Create a GIF or video for a given chart URL.
+"""Create a GIF or video for a given chart URL."""
 
-"""
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
 from urllib.parse import parse_qs, urlencode, urlparse, urlunparse

apps/cli/__init__.py

@@ -167,7 +167,8 @@ def cli_back() -> None:
                 "update": "apps.step_update.cli.cli",
                 "archive": "apps.step_update.cli.archive_cli",
                 "explorer-update": "apps.explorer_update.cli.cli",
-                "pr": "apps.utils.draft_pull_request.cli",
+                "prr": "apps.utils.draft_pull_request.cli",
+                "pr": "apps.pr.cli.cli",
             },
         },
         {

apps/metadata_migrate/cli.py

@@ -209,9 +209,9 @@ def cli(
 
         # empty fields to be filled
         vars[col]["presentation"] = {}
-        vars[col]["presentation"][
-            "title_public"
-        ] = "TBD - Indicator title to be shown in data pages, that overrides the indicator's title."
+        vars[col]["presentation"]["title_public"] = (
+            "TBD - Indicator title to be shown in data pages, that overrides the indicator's title."
+        )
         vars[col]["presentation"]["title_variant"] = "TBD - Indicator's title variant"
         vars[col]["presentation"]["attribution_short"] = "TBD - Indicator's attribution (shorter version)"
         vars[col]["presentation"]["faqs"] = [

apps/metagpt/cli.py

@@ -1,4 +1,5 @@
 """Client module."""
+
 import os
 from typing import Any, Dict, Literal, cast
 

apps/metagpt/prompts.py

@@ -2,6 +2,7 @@
 
 Contains instructions to correctly query chat GPT for the different use cases (snapshot, garden, grapher, etc.). This includes details on what each metadata field means.
 """
+
 import json
 from typing import Any, Dict, List
 

apps/metagpt/utils.py

@@ -1,4 +1,5 @@
 """Auxiliary utils for metagpt app."""
+
 from pathlib import Path
 from typing import Dict, List
 

apps/owidbot/cli.py

@@ -9,7 +9,7 @@
 from rich_click.rich_command import RichCommand
 
 from apps.owidbot import anomalist, chart_diff, data_diff, grapher
-from etl.config import get_container_name
+from etl.config import OWIDBOT_ACCESS_TOKEN, get_container_name
 
 from . import github_utils as gh_utils
 
@@ -57,7 +57,7 @@ def cli(
     if repo_name not in get_args(REPOS):
         raise AssertionError("Invalid repo")
 
-    repo = gh_utils.get_repo(repo_name)
+    repo = gh_utils.get_repo(repo_name, access_token=OWIDBOT_ACCESS_TOKEN)
     pr = gh_utils.get_pr(repo, branch)
     if pr is None:
         log.warning(f"No open PR found for branch {branch}")

apps/owidbot/github_utils.py

@@ -17,9 +17,12 @@
 
 def get_repo(repo_name: str, access_token: Optional[str] = None) -> github.Repository.Repository:
     if not access_token:
-        assert config.OWIDBOT_ACCESS_TOKEN, "OWIDBOT_ACCESS_TOKEN is not set"
-        access_token = config.OWIDBOT_ACCESS_TOKEN
-    auth = Auth.Token(access_token)
+        # Don't auth, be aware that you won't be able to do write operations. You should
+        # set up your access token on https://github.com/settings/tokens.
+        auth = None
+    else:
+        auth = Auth.Token(access_token)
+
     g = Github(auth=auth)
     return g.get_repo(f"owid/{repo_name}")
 
@@ -96,6 +99,37 @@ def compute_git_blob_sha1(content: bytes) -> str:
     return sha1.hexdigest()
 
 
+def _github_access_token():
+    # Use GITHUB_TOKEN if set, otherwise use OWIDBOT_ACCESS_TOKEN
+    if config.GITHUB_TOKEN:
+        return config.GITHUB_TOKEN
+    elif config.OWIDBOT_ACCESS_TOKEN:
+        return config.OWIDBOT_ACCESS_TOKEN
+    else:
+        raise AssertionError("You need to set GITHUB_TOKEN or OWIDBOT_ACCESS_TOKEN in your .env file to commit.")
+
+
+def create_branch_if_not_exists(repo_name: str, branch: str, dry_run: bool) -> None:
+    """Create a branch if it doesn't exist."""
+    repo = get_repo(repo_name, access_token=_github_access_token())
+    try:
+        repo.get_branch(branch)
+    except github.GithubException as e:
+        if e.status == 404:
+            if not dry_run:
+                try:
+                    master_ref = repo.get_branch("main").commit.sha
+                    log.info(f"Using 'main' branch as reference for creating {branch}.")
+                except github.GithubException:
+                    master_ref = repo.get_branch("master").commit.sha
+                    log.info(f"Using 'master' branch as reference for creating {branch}.")
+                log.info(f"Creating branch {branch} with reference {master_ref}.")
+                repo.create_git_ref(ref=f"refs/heads/{branch}", sha=master_ref)
+            log.info(f"Branch {branch} created in {repo.name}.")
+        else:
+            raise e
+
+
 def commit_file_to_github(
     content: str,
     repo_name: str,
@@ -106,7 +140,7 @@ def commit_file_to_github(
 ) -> None:
     """Commit a table to a GitHub repository using the GitHub API."""
     # Get the repository object
-    repo = get_repo(repo_name)
+    repo = get_repo(repo_name, access_token=_github_access_token())
     new_content_checksum = compute_git_blob_sha1(content.encode("utf-8"))
 
     try:

apps/pr/categories.py

@@ -0,0 +1,69 @@
+# Add EMOJIs for each PR type
+PR_CATEGORIES = {
+    "data": {
+        "emoji": "📊",
+        "emoji_raw": ":bar_chart:",
+        "description": "data update or addition",
+    },
+    "bug": {
+        "emoji": "🐛",
+        "emoji_raw": ":bug:",
+        "description": "bug fix for the user",
+    },
+    "refactor": {
+        "emoji": "🔨",
+        "emoji_raw": ":hammer:",
+        "description": "a code change that neither fixes a bug nor adds a feature for the user",
+    },
+    "enhance": {
+        "emoji": "✨",
+        "emoji_raw": ":sparkles:",
+        "description": "visible improvement over a current implementation without adding a new feature or fixing a bug",
+    },
+    "feature": {
+        "emoji": "🎉",
+        "emoji_raw": ":tada:",
+        "description": "new feature for the user",
+    },
+    "docs": {
+        "emoji": "📜",
+        "emoji_raw": ":scroll:",
+        "description": "documentation only changes",
+        "shortcut_key": "0",
+    },
+    "chore": {
+        "emoji": "🐝",
+        "emoji_raw": ":honeybee:",
+        "description": "upgrading dependencies, tooling, etc. No production code change",
+    },
+    "style": {
+        "emoji": "💄",
+        "emoji_raw": ":lipstick:",
+        "description": "formatting, missing semi colons, etc. No production code change",
+    },
+    "wip": {
+        "emoji": "🚧",
+        "emoji_raw": ":construction:",
+        "description": "work in progress - intermediate commits that will be explained later on",
+    },
+    "tests": {
+        "emoji": "✅",
+        "emoji_raw": ":white_check_mark:",
+        "description": "adding missing tests, refactoring tests, etc. No production code change",
+    },
+}
+PR_CATEGORIES_MD_DESCRIPTION = "- " + "\n- ".join(
+    f"**{choice}**: {choice_params['description']}" for choice, choice_params in PR_CATEGORIES.items()
+)
+PR_CATEGORIES_CHOICES = [
+    {
+        "title": f"{v['emoji']} {k}",
+        "value": k,
+        "shortcut_key": v.get("shortcut_key", k[0]),
+    }
+    for k, v in PR_CATEGORIES.items()
+]
+PR_CATEGORIES_CHOICES = sorted(PR_CATEGORIES_CHOICES, key=lambda x: x["shortcut_key"])
+assert len(set([x["shortcut_key"].lower() for x in PR_CATEGORIES_CHOICES])) == len(
+    PR_CATEGORIES_CHOICES
+), "Shortcut keys must be unique"